dolphinscheduler最新版2.0.2+mysql8.0.16部署（一）

1. 下載dolphinscheduler

下載地址https://dolphinscheduler.apache.org/en-us/download/download.html

下載版本為：src源碼，需要重新編譯

查看pom.xml對應的組件版本，這里不介紹編譯方式，下次用到介紹

  1 <curator.version>4.3.0</curator.version>
  2 
  3 <zookeeper.version>3.4.14</zookeeper.version>
  4 
  5 <spring.version>5.3.12</spring.version>
  6 
  7 <spring.boot.version>2.5.6</spring.boot.version>
  8 
  9 <java.version>1.8</java.version>
 10 
 11 <logback.version>1.2.3</logback.version>
 12 
 13 <hadoop.version>2.7.3</hadoop.version>
 14 
 15 <quartz.version>2.3.0</quartz.version>
 16 
 17 <jackson.version>2.10.5</jackson.version>
 18 
 19 <mybatis-plus.version>3.2.0</mybatis-plus.version>
 20 
 21 <mybatis.spring.version>2.0.1</mybatis.spring.version>
 22 
 23 <cron.utils.version>9.1.3</cron.utils.version>
 24 
 25 <druid.version>1.2.4</druid.version>
 26 
 27 <h2.version>1.4.200</h2.version>
 28 
 29 <commons.codec.version>1.11</commons.codec.version>
 30 
 31 <commons.logging.version>1.1.1</commons.logging.version>
 32 
 33 <httpclient.version>4.4.1</httpclient.version>
 34 
 35 <httpcore.version>4.4.1</httpcore.version>
 36 
 37 <junit.version>4.12</junit.version>
 38 
 39 <mysql.connector.version>8.0.16</mysql.connector.version>
 40 
 41 <slf4j.api.version>1.7.5</slf4j.api.version>
 42 
 43 <slf4j.log4j12.version>1.7.5</slf4j.log4j12.version>
 44 
 45 <commons.collections.version>3.2.2</commons.collections.version>
 46 
 47 <commons.httpclient>3.0.1</commons.httpclient>
 48 
 49 <commons.beanutils.version>1.9.4</commons.beanutils.version>
 50 
 51 <commons.configuration.version>1.10</commons.configuration.version>
 52 
 53 <commons.lang.version>2.6</commons.lang.version>
 54 
 55 <commons.email.version>1.5</commons.email.version>
 56 
 57 <poi.version>4.1.2</poi.version>
 58 
 59 <javax.servlet.api.version>3.1.0</javax.servlet.api.version>
 60 
 61 <commons.collections4.version>4.1</commons.collections4.version>
 62 
 63 <guava.version>24.1-jre</guava.version>
 64 
 65 <postgresql.version>42.2.5</postgresql.version>
 66 
 67 <hive.jdbc.version>2.1.0</hive.jdbc.version>
 68 
 69 <commons.io.version>2.4</commons.io.version>
 70 
 71 <oshi.core.version>3.9.1</oshi.core.version>
 72 
 73 <clickhouse.jdbc.version>0.1.52</clickhouse.jdbc.version>
 74 
 75 <mssql.jdbc.version>6.1.0.jre8</mssql.jdbc.version>
 76 
 77 <presto.jdbc.version>0.238.1</presto.jdbc.version>
 78 
 79 <spotbugs.version>3.1.12</spotbugs.version>
 80 
 81 <checkstyle.version>3.1.2</checkstyle.version>
 82 
 83 <zookeeper.version>3.4.14</zookeeper.version>
 84 
 85 <curator.test>2.12.0</curator.test>
 86 
 87 <frontend-maven-plugin.version>1.6</frontend-maven-plugin.version>
 88 
 89 <maven-compiler-plugin.version>3.3</maven-compiler-plugin.version>
 90 
 91 <maven-assembly-plugin.version>3.1.0</maven-assembly-plugin.version>
 92 
 93 <maven-release-plugin.version>2.5.3</maven-release-plugin.version>
 94 
 95 <maven-javadoc-plugin.version>2.10.3</maven-javadoc-plugin.version>
 96 
 97 <maven-source-plugin.version>2.4</maven-source-plugin.version>
 98 
 99 <maven-surefire-plugin.version>2.22.1</maven-surefire-plugin.version>
100 
101 <maven-dependency-plugin.version>3.1.1</maven-dependency-plugin.version>
102 
103 <rpm-maven-plugion.version>2.2.0</rpm-maven-plugion.version>
104 
105 <jacoco.version>0.8.4</jacoco.version>
106 
107 <jcip.version>1.0</jcip.version>
108 
109 <maven.deploy.skip>false</maven.deploy.skip>
110 
111 <cobertura-maven-plugin.version>2.7</cobertura-maven-plugin.version>
112 
113 <mockito.version>2.21.0</mockito.version>
114 
115 <powermock.version>2.0.2</powermock.version>
116 
117 <servlet-api.version>2.5</servlet-api.version>
118 
119 <swagger.version>1.9.3</swagger.version>
120 
121 <springfox.version>2.9.2</springfox.version>
122 
123 <swagger-models.version>1.5.24</swagger-models.version>
124 
125 <guava-retry.version>2.0.0</guava-retry.version>
126 
127 <protostuff.version>1.7.2</protostuff.version>
128 
129 <reflections.version>0.9.12</reflections.version>
130 
131 <byte-buddy.version>1.9.16</byte-buddy.version>
132 
133 <java-websocket.version>1.5.1</java-websocket.version>
134 
135 <py4j.version>0.10.9</py4j.version>
136 
137 <auto-service.version>1.0.1</auto-service.version>
138 
139 <jacoco.skip>false</jacoco.skip>
140 
141 <netty.version>4.1.53.Final</netty.version>
142 
143 <maven-jar-plugin.version>3.2.0</maven-jar-plugin.version>

View Code

以dolphin-server為server機，以下需要在每臺機器創建用戶2. 創建應用用戶及免密

# 創建用戶需使用root登錄，設置部署用戶名，請自行修改，后面以dolphinscheduler為例

useradd dolphin;

# 設置用戶密碼，請自行修改，后面以dolphinscheduler123為例

echo "dolphin" | passwd --stdin dolphin

# 配置sudo免密

echo 'dolphin ALL=(ALL) NOPASSWD: NOPASSWD: ALL' >> /etc/sudoers

在84機器上執行

#add ip hostname

vi /etc/hosts

198.28.1.82 ht-28-tdh82

198.28.1.83 ht-28-tdh83

198.28.1.84 ht-28-tdh84

#同步dolphin-server上的/etc/hosts到所有部署機器

for ip in dolphin-82 dolphin-83;

sudo scp -r /etc/hosts $ip:/etc/ #在運行中需要輸入root密碼

done

#在dolphin-84上，切換到部署用戶并配置ssh本機免密登錄

su - dolphin;

ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa

cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

chmod 600 ~/.ssh/authorized_keys

for ip in ht-28-tdh82 ht-28-tdh83; #請將此處ds2 ds3替換為自己要部署的機器的hostname

ssh-copy-id $ip #該操作執行過程中需要手動輸入dolphinscheduler用戶的密碼

done

3. Mysql部署

下載mysql 版本 (mysql-8.0.16-linux-glibc2.12-x86_64.tar)

cd /opt/dolphin

tar -xvf mysql-8.0.16-linux-glibc2.12-x86_64.tar

tar -xvJf mysql-8.0.16-linux-glibc2.12-x86_64.tar.xz

# 建立軟鏈接，便于以后版本升級

ln -s mysql-8.0.16-linux-glibc2.12-x86_64 mysql

mkdir -p /opt/dolphin/mysql/3306/data

mkdir -p /opt/dolphin/mysql/3306/log

mkdir -p /opt/dolphin/mysql/3306/tmp

# 創建配置文件-root下

cd /etc

# 在my.cnf文件中添加對應的配置項

cp my.cnf my.cnf.bak20220125
vi my.cnf

 1 [client] # 客戶端設置，即客戶端默認的連接參數
 2 
 3 port = 3306 # 默認連接端口
 4 
 5 socket = /opt/dolphin/mysql/3306/tmp/mysql.sock # 用于本地連接的socket套接字，mysqld守護進程生成了這個文件
 6 
 7 [mysqld] # 服務端基本設置
 8 
 9 # 基礎設置
10 
11 server-id = 1 # Mysql服務的唯一編號 每個mysql服務Id需唯一
12 
13 port = 3306 # MySQL監聽端口
14 
15 basedir = /opt/dolphin/mysql # MySQL安裝根目錄
16 
17 datadir = /opt/dolphin/mysql/3306/data # MySQL數據文件所在位置
18 
19 tmpdir = /opt/dolphin/mysql/3306/tmp # 臨時目錄，比如load data infile會用到
20 
21 socket = /opt/dolphin/mysql/3306/tmp/mysql.sock # 為MySQL客戶端程序和服務器之間的本地通訊指定一個套接字文件
22 
23 pid-file = /opt/dolphin/mysql/3306/log/mysql.pid # pid文件所在目錄
24 
25 #skip_name_resolve = 1 # 只能用IP地址檢查客戶端的登錄，不用主機名
26 
27 character-set-server = utf8mb4 # 數據庫默認字符集,主流字符集支持一些特殊表情符號（特殊表情符占用4個字節）
28 
29 transaction_isolation = READ-COMMITTED # 事務隔離級別，默認為可重復讀，MySQL默認可重復讀級別
30 
31 collation-server = utf8mb4_general_ci # 數據庫字符集對應一些排序等規則，注意要和character-set-server對應
32 
33 init_connect='SET NAMES utf8mb4' # 設置client連接mysql時的字符集,防止亂碼
34 
35 lower_case_table_names = 1 # 是否對sql語句大小寫敏感，1表示不敏感
36 
37 max_connections = 400 # 最大連接數
38 
39 max_connect_errors = 1000 # 最大錯誤連接數
40 
41 explicit_defaults_for_timestamp = true # TIMESTAMP如果沒有顯示聲明NOT NULL，允許NULL值
42 
43 max_allowed_packet = 128M # SQL數據包發送的大小，如果有BLOB對象建議修改成1G
44 
45 interactive_timeout = 1800 # MySQL連接閑置超過一定時間后(單位：秒)將會被強行關閉
46 
47 wait_timeout = 1800 # MySQL默認的wait_timeout值為8個小時, interactive_timeout參數需要同時配置才能生效
48 
49 tmp_table_size = 16M # 內部內存臨時表的最大值 ，設置成128M；比如大數據量的group by ,order by時可能用到臨時表；超過了這個值將寫入磁盤，系統IO壓力增大

View Code

# 進入MySQL的bin目錄安裝MySQL數據庫

cd /opt/dolphin/mysql/bin

# 初始化數據庫，并指定啟動mysql的用戶

./mysqld --initialize --user=dolphin

生成隨機密碼

root@localhost: ,8v_<fe;,.cP

設置開機自啟動服務

# 復制啟動腳本到資源目錄

cp /opt/dolphin/mysql/support-files/mysql.server /etc/rc.d/init.d/mysqld

# 增加mysqld服務控制腳本執行權限

chmod +x /etc/rc.d/init.d/mysqld

# 將mysqld服務加入到系統服務

chkconfig --add mysqld

# 檢查mysqld服務是否已經生效

chkconfig --list mysqld

# 切換至mysql用戶，啟動mysql

su - dolphin

service mysqld start

配置環境變量

# 修改配置文件，增加

cp .bash_profile .bash_profile_bak20220125

vi .bash_profile

export PATH=$PATH:/opt/dolphin/mysql/bin

# 立即生效

source .bash_profile

登陸，修改密碼

# 登陸mysql

./mysql -uroot -p -S/opt/dolphin/mysql/3306/tmp/mysql.sock

報錯：ERROR 2059 (HY000): Authentication plugin 'caching_sha2_password' cannot be loaded: /usr/lib64/mysql/plugin/caching_sha2_password.so: cannot open shared object file: No such file or directory

修改

vi /etc/my.cnf

[mysqld]

default_authentication_plugin=mysql_native_password

重啟mysql

service mysqld restart

# 修改root用戶密碼

alter user 'root'@'localhost'IDENTIFIED BY '123456';

4. 數據庫初始化

· 進入數據庫，默認數據庫是PostgreSQL，如選擇Mysql的話，后續需要添加mysql-connector-java驅動包到DolphinScheduler的lib目錄下，這里以mysql為例

mysql -uroot -p

· 進入數據庫命令行窗口后，執行數據庫初始化命令，設置訪問賬號和密碼。

mysql> CREATE DATABASE dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;

Query OK, 1 row affected (0.00 sec)

mysql> create user 'dolphin'@'localhost' identified by 'dolphin';

mysql> GRANT ALL PRIVILEGES ON dolphinscheduler.* TO 'dolphin'@'localhost';

Query OK, 0 rows affected, 1 warning (0.00 sec)

mysql> flush privileges;

Query OK, 0 rows affected (0.00 sec)

5. Ds部署

root下面創建目錄

mkdir -p /opt/dolphin;

賦權

chown -R dolphin:dolphin /opt/dolphin

cd /opt/dolphin;

tar -zxvf apache-dolphinscheduler-2.0.2-bin.tar.gz -C /opt/dolphin;

#在ds1上，修改目錄權限，使得部署用戶對dolphin-backend目錄有操作權限

chown -R dolphin:dolphin apache-dolphinscheduler-2.0.2-bin

· 修改 conf 目錄下 application-mysql.yaml 中的下列配置

vi conf/application-mysql.yaml

 1 spring:
 2 
 3 datasource:
 4 
 5 driver-class-name: com.mysql.jdbc.Driver
 6 
 7 url: jdbc:mysql://198.28.1.84:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
 8 
 9 username: dolphin
10 
11 password: dolphin
12 
13 hikari:
14 
15 connection-test-query: select 1
16 
17 minimum-idle: 5
18 
19 auto-commit: true
20 
21 validation-timeout: 3000
22 
23 pool-name: DolphinScheduler
24 
25 maximum-pool-size: 50
26 
27 connection-timeout: 30000
28 
29 idle-timeout: 600000
30 
31 leak-detection-threshold: 0
32 
33 initialization-fail-timeout: 1

View Code

還需要手動添加 [?mysql-connector-java 驅動 jar?] 包到 lib 目錄下，這里下載的是mysql-connector-java-8.0.16.jar

· 修改并保存完后，執行 script 目錄下的創建表及導入基礎數據腳本

sh script/create-dolphinscheduler.sh

發現報錯，報出來數據庫不對為h2，原因是因為初始化數據腳本是按照

/opt/dolphin/apache-dolphinscheduler-2.0.2-bin/conf/config/install_config.conf

去讀取配置

可以看下create-dolphinscheduler.sh文件

里面加載了install_config.conf這個文件

  1 #
  2 
  3 # Licensed to the Apache Software Foundation (ASF) under one or more
  4 
  5 # contributor license agreements. See the NOTICE file distributed with
  6 
  7 # this work for additional information regarding copyright ownership.
  8 
  9 # The ASF licenses this file to You under the Apache License, Version 2.0
 10 
 11 # (the "License"); you may not use this file except in compliance with
 12 
 13 # the License. You may obtain a copy of the License at
 14 
 15 #
 16 
 17 # http://www.apache.org/licenses/LICENSE-2.0
 18 
 19 #
 20 
 21 # Unless required by applicable law or agreed to in writing, software
 22 
 23 # distributed under the License is distributed on an "AS IS" BASIS,
 24 
 25 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 26 
 27 # See the License for the specific language governing permissions and
 28 
 29 # limitations under the License.
 30 
 31 #
 32 
 33 # ---------------------------------------------------------
 34 
 35 # INSTALL MACHINE
 36 
 37 # ---------------------------------------------------------
 38 
 39 # A comma separated list of machine hostname or IP would be installed DolphinScheduler,
 40 
 41 # including master, worker, api, alert. If you want to deploy in pseudo-distributed
 42 
 43 # mode, just write a pseudo-distributed hostname
 44 
 45 # Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5"
 46 
 47 ips="ht-28-tdh82,ht-28-tdh83"
 48 
 49 # Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine
 50 
 51 # modify it if you use different ssh port
 52 
 53 sshPort="22"
 54 
 55 # A comma separated list of machine hostname or IP would be installed Master server, it
 56 
 57 # must be a subset of configuration `ips`.
 58 
 59 # Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2"
 60 
 61 masters="ht-28-tdh84"
 62 
 63 # A comma separated list of machine <hostname>:<workerGroup> or <IP>:<workerGroup>.All hostname or IP must be a
 64 
 65 # subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts
 66 
 67 # Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default"
 68 
 69 workers="ht-28-tdh82:default,ht-28-tdh83:default"
 70 
 71 # A comma separated list of machine hostname or IP would be installed Alert server, it
 72 
 73 # must be a subset of configuration `ips`.
 74 
 75 # Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3"
 76 
 77 alertServer="ht-28-tdh84"
 78 
 79 # A comma separated list of machine hostname or IP would be installed API server, it
 80 
 81 # must be a subset of configuration `ips`.
 82 
 83 # Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1"
 84 
 85 apiServers="ht-28-tdh84"
 86 
 87 # A comma separated list of machine hostname or IP would be installed Python gateway server, it
 88 
 89 # must be a subset of configuration `ips`.
 90 
 91 # Example for hostname: pythonGatewayServers="ds1", Example for IP: pythonGatewayServers="192.168.8.1"
 92 
 93 pythonGatewayServers="ht-28-tdh84"
 94 
 95 # The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists.
 96 
 97 # Do not set this configuration same as the current path (pwd)
 98 
 99 installPath="/opt/dolphin/apache-dolphinscheduler-2.0.2-bin"
100 
101 # The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh`
102 
103 # script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs
104 
105 # to be created by this user
106 
107 deployUser="dolphin"
108 
109 # The directory to store local data for all machine we config above. Make sure user `deployUser` have permissions to read and write this directory.
110 
111 dataBasedirPath="/opt/dolphin/apache-dolphinscheduler-2.0.2-bin/tmp"
112 
113 # ---------------------------------------------------------
114 
115 # DolphinScheduler ENV
116 
117 # ---------------------------------------------------------
118 
119 # JAVA_HOME, we recommend use same JAVA_HOME in all machine you going to install DolphinScheduler
120 
121 # and this configuration only support one parameter so far.
122 
123 javaHome="/usr/java/jdk1.8.0_144"
124 
125 # DolphinScheduler API service port, also this is your DolphinScheduler UI component's URL port, default value is 12345
126 
127 apiServerPort="12345"
128 
129 # ---------------------------------------------------------
130 
131 # Database
132 
133 # NOTICE: If database value has special characters, such as `.*[]^${}\+?|()@#&`, Please add prefix `\` for escaping.
134 
135 # ---------------------------------------------------------
136 
137 # The type for the metadata database
138 
139 # Supported values: ``postgresql``, ``mysql`, `h2``.
140 
141 DATABASE_TYPE=${DATABASE_TYPE:-"mysql"}
142 
143 # Spring datasource url, following <HOST>:<PORT>/<database>?<parameter> format, If you using mysql, you could use jdbc
144 
145 # string jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 as example
146 
147 SPRING_DATASOURCE_URL=${SPRING_DATASOURCE_URL:-"jdbc:mysql://localhost:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8"}
148 
149 # Spring datasource username
150 
151 SPRING_DATASOURCE_USERNAME=${SPRING_DATASOURCE_USERNAME:-"dolphin"}
152 
153 # Spring datasource password
154 
155 SPRING_DATASOURCE_PASSWORD=${SPRING_DATASOURCE_PASSWORD:-"dolphin"}
156 
157 # ---------------------------------------------------------
158 
159 # Registry Server
160 
161 # ---------------------------------------------------------
162 
163 # Registry Server plugin name, should be a substring of `registryPluginDir`, DolphinScheduler use this for verifying configuration consistency
164 
165 registryPluginName="zookeeper"
166 
167 # Registry Server address.
168 
169 registryServers="198.28.1.81:2181,198.28.1.82:2181,198.28.1.83:2181"
170 
171 # The root of zookeeper, for now DolphinScheduler default registry server is zookeeper.
172 
173 zkRoot="/dolphinscheduler"
174 
175 # ---------------------------------------------------------
176 
177 # Worker Task Server
178 
179 # ---------------------------------------------------------
180 
181 # Worker Task Server plugin dir. DolphinScheduler will find and load the worker task plugin jar package from this dir.
182 
183 taskPluginDir="lib/plugin/task"
184 
185 # resource storage type: HDFS, S3, NONE
186 
187 resourceStorageType="NONE"
188 
189 # resource store on HDFS/S3 path, resource file will store to this hdfs path, self configuration, please make sure the directory exists on hdfs and has read write permissions.
190 
191 "/dolphinscheduler" is recommended
192 
193 resourceUploadPath="/dolphinscheduler"
194 
195 # if resourceStorageType is HDFS，defaultFS write namenode address，HA, you need to put core-site.xml and hdfs-site.xml in the conf directory.
196 
197 # if S3，write S3 address，HA，for example ：s3a://dolphinscheduler，
198 
199 # Note，S3 be sure to create the root directory /dolphinscheduler
200 
201 defaultFS="hdfs://198.28.1.81:8020"
202 
203 # if resourceStorageType is S3, the following three configuration is required, otherwise please ignore
204 
205 s3Endpoint="http://192.168.xx.xx:9010"
206 
207 s3AccessKey="xxxxxxxxxx"
208 
209 s3SecretKey="xxxxxxxxxx"
210 
211 # resourcemanager port, the default value is 8088 if not specified
212 
213 resourceManagerHttpAddressPort="8088"
214 
215 # if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single node, keep this value empty
216 
217 yarnHaIps="192.168.xx.xx,192.168.xx.xx"
218 
219 # if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single node, you only need to replace 'yarnIp1' to actual re
220 
221 sourcemanager hostname
222 
223 singleYarnIp="yarnIp1"
224 
225 # who has permission to create directory under HDFS/S3 root path
226 
227 # Note: if kerberos is enabled, please config hdfsRootUser=
228 
229 hdfsRootUser="hdfs"
230 
231 # kerberos config
232 
233 # whether kerberos starts, if kerberos starts, following four items need to config, otherwise please ignore
234 
235 kerberosStartUp="false"
236 
237 # kdc krb5 config file path
238 
239 krb5ConfPath="$installPath/conf/krb5.conf"
240 
241 # keytab username,watch out the @ sign should followd by \\
242 
243 keytabUserName="hdfs"
244 
245 # username keytab path
246 
247 keytabPath="$installPath/conf/hdfs.keytab"
248 
249 # kerberos expire time, the unit is hour
250 
251 kerberosExpireTime="2"
252 
253 # use sudo or not
254 
255 sudoEnable="true"
256 
257 # worker tenant auto create
258 
259 workerTenantAutoCreate="false"

View Code

環境變量env/dolphinscheduler_env.sh

 1 #
 2 
 3 # Licensed to the Apache Software Foundation (ASF) under one or more
 4 
 5 # contributor license agreements. See the NOTICE file distributed with
 6 
 7 # this work for additional information regarding copyright ownership.
 8 
 9 # The ASF licenses this file to You under the Apache License, Version 2.0
10 
11 # (the "License"); you may not use this file except in compliance with
12 
13 # the License. You may obtain a copy of the License at
14 
15 #
16 
17 # http://www.apache.org/licenses/LICENSE-2.0
18 
19 #
20 
21 # Unless required by applicable law or agreed to in writing, software
22 
23 # distributed under the License is distributed on an "AS IS" BASIS,
24 
25 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
26 
27 # See the License for the specific language governing permissions and
28 
29 # limitations under the License.
30 
31 #
32 
33 export HADOOP_HOME=/opt/soft/hadoop
34 
35 export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
36 
37 export SPARK_HOME1=/opt/soft/spark1
38 
39 export SPARK_HOME2=/opt/soft/spark2
40 
41 export PYTHON_HOME=/opt/soft/python
42 
43 export JAVA_HOME=/usr/java/jdk1.8.0_144
44 
45 export HIVE_HOME=/opt/soft/hive
46 
47 export FLINK_HOME=/opt/soft/flink
48 
49 export DATAX_HOME=/opt/soft/datax
50 
51 export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH