yarn rmadmin -getServiceState rm1 #查看状态 
yarn rmadmin -getServiceState rm2 #查看状态

dfsadmin -setBalancerBandwidth 10485760
hdfs balancer 

/home/hadoop/soft/presto/bin/launcher start

cd /home/hadoop/soft/azkaban-web-2.5.0/
nohup bin/azkaban-web-start.sh > /home/hadoop/soft/azkaban-web-2.5.0/webServer.log 2>&1 &

cd  /home/hadoop/soft/azkaban-executor-2.5.0/
nohup bin/azkaban-executor-start.sh > /home/hadoop/soft/azkaban-executor-2.5.0/executorServer.log 2>&1 &

cd /home/hadoop/soft/kylin/bin
sh kylin.sh start

cd  /home/hadoop/soft/hue
nohup build/env/bin/supervisor &

vim /home/hadoop/soft/hadoop/etc/hadoop/yarn-site.xml
<property>
  <name>yarn.resourcemanager.scheduler.class</name>
  <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value>
</property>

export SPARK_MASTER_WEBUI_PORT=8081
export SCALA_HOME=/home/hadoop/soft/scala
export JAVA_HOME=/home/hadoop/soft/jdk
export HADOOP_HOME=/home/hadoop/soft/hadoop
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
SPARK_MASTER_IP=master1
SPARK_LOCAL_DIRS=/home/hadoop/soft/spark
SPARK_DRIVER_MEMORY=4G

spark.master                        yarn
spark.home                          /home/hadoop/soft/spark
spark.eventLog.enabled              true
spark.eventLog.dir                  hdfs://ns1/spark/spark-log
spark.serializer                    org.apache.spark.serializer.KryoSerializer
# hive on spark 建议5/6/7
spark.executor.cores                6
# 按6*12G/12vcores计算
spark.executor.memory               5222m
# 28个计算节点*每个节点2个executor
spark.executor.instances            56
# 所有core的两三倍：56*6*3=1008
spark.default.parallelism           1000
# 15% * spark.executor.memory
spark.yarn.executor.memoryOverhead  921m
spark.driver.memory                 4g
spark.yarn.driver.memoryOverhead    400m
spark.yarn.jars                     hdfs://ns1/spark/jars/*.jar

ln -s ~/soft/spark/conf/spark-defaults.conf ~/soft/hive/conf/

./bin/spark-submit --class org.apache.spark.examples.SparkPi --master yarn --deploy-mode client ./examples/jars/spark-examples_2.11-2.2.0.jar 10

ln -s /home/hadoop/soft/spark/jars/scala-library-2.11.8.jar /home/hadoop/soft/hive/lib/
ln -s /home/hadoop/soft/spark/jars/spark-network-common_2.11-2.2.0.jar /home/hadoop/soft/hive/lib/
ln -s /home/hadoop/soft/spark/jars/spark-core_2.11-2.2.0hiv.jar /home/hadoop/soft/hive/lib/

mkdir hive-jars
mv /home/hadoop/soft/spark/jars/hive*.jar /home/hadoop/soft/spark/hive-jars

# online machine: pwd --> /root
mkdir -p /root/download
yum install --downloadonly --downloaddir=/root/download <package-name>
tar zcf download.tar.gz download/

# offline machine: pwd --> /root
tar xf download.tar.gz
yum localinstall download/*

--> Processing Dependency: rpm = 4.11.3-25.el7 for package: rpm-libs-4.11.3-25.el7.x86_64
--> Finished Dependency Resolution
Error: Package: 1:net-snmp-agent-libs-5.7.2-24.el7.i686 (@Server)
           Requires: librpm.so.3
           Removing: rpm-libs-4.11.3-17.el7.i686 (@Server)
               librpm.so.3
           Updated By: rpm-libs-4.11.3-25.el7.x86_64 (/rpm-libs-4.11.3-25.el7.x86_64)
               Not found
Error: Package: rpm-build-libs-4.11.3-17.el7.x86_64 (@Server)
           Requires: rpm-libs(x86-64) = 4.11.3-17.el7
           Removing: rpm-libs-4.11.3-17.el7.x86_64 (@Server)
               rpm-libs(x86-64) = 4.11.3-17.el7
           Updated By: rpm-libs-4.11.3-25.el7.x86_64 (/rpm-libs-4.11.3-25.el7.x86_64)
               rpm-libs(x86-64) = 4.11.3-25.el7
Error: Package: 1:net-snmp-agent-libs-5.7.2-24.el7.i686 (@Server)
           Requires: librpmio.so.3
           Removing: rpm-libs-4.11.3-17.el7.i686 (@Server)
               librpmio.so.3
           Updated By: rpm-libs-4.11.3-25.el7.x86_64 (/rpm-libs-4.11.3-25.el7.x86_64)
               Not found
Error: Package: rpm-libs-4.11.3-25.el7.x86_64 (/rpm-libs-4.11.3-25.el7.x86_64)
           Requires: rpm = 4.11.3-25.el7
           Installed: rpm-4.11.3-17.el7.x86_64 (@Server)
               rpm = 4.11.3-17.el7
 You could try using --skip-broken to work around the problem
 You could try running: rpm -Va --nofiles --nodigest

docker run --name mysql --restart=always \
-v /home/tony/mysql_data:/var/lib/mysql \
-p 3306:3306 \
-e MYSQL_ROOT_PASSWORD=huawei123 \
-d mysql:5.6

docker run --name zabbix-server-mysql --restart always \
--link mysql:mysql_host \
-p 10051:10051 \
-e DB_SERVER_HOST="mysql_host" \
-e MYSQL_USER="root" \
-e MYSQL_PASSWORD="huawei123" \
-v /home/tony/zabbix/externalscripts:/usr/lib/zabbix/externalscripts \
-d zabbix/zabbix-server-mysql

docker run --name zabbix-web-nginx-mysql \
--restart always \
--link mysql:mysql_host \
--link zabbix-server-mysql:zabbix-server-mysql \
-p 81:80 \
-e DB_SERVER_HOST="mysql_host" \
-e MYSQL_USER="root" \
-e MYSQL_PASSWORD="huawei123" \
-e ZBX_SERVER_HOST="zabbix-server-mysql" \
-e PHP_TZ="Asia/Shanghai" \
-v /home/tony/zabbix/msyh.ttf:/usr/share/zabbix/fonts/graphfont.ttf \
-d zabbix/zabbix-web-nginx-mysql:latest

#!/bin/sh

#--------------------------------------------------------------------------------------------
# Expecting the following arguments in order -
# <host> = hostname/ip-address of Hadoop cluster NameNode server.
#        This is made available as a macro in host configuration.
# <port> = Port # on which the NameNode metrics are available (default = 50070)
#        This is made available as a macro in host configuration.
# <name_in_zabbix> = Name by which the Hadoop NameNode is configured in Zabbix.
#        This is made available as a macro in host configuration.
# <monitor_type> is the parameter (NN or DFS) you want to monitor.
#--------------------------------------------------------------------------------------------

COMMAND_LINE="$0 $*" 
export SCRIPT_NAME="$0"

usage() {
   echo "Usage: $SCRIPT_NAME <host> <port> <name_in_zabbix> <monitor_type>"
}

if [ $# -ne 4 ]
then
    usage ;
    exit ;
fi


#--------------------------------------------------------------------------------------------
# First 2 parameters are required for connecting to Hadoop NameNode
# The 3th parameter HADOOP_NAME_IN_ZABBIX is required to be sent back to Zabbix to identify the 
# Zabbix host/entity for which these metrics are destined.
# The 4th parameter MONITOR_TYPE is to specify the type to monitoring(NN or DFS)
#--------------------------------------------------------------------------------------------
export CLUSTER_HOST=$1
export METRICS_PORT=$2
export HADOOP_NAME_IN_ZABBIX=$3
export MONITOR_TYPE=$4

#--------------------------------------------------------------------------------------------
# Set the data output file and the log fle from zabbix_sender
#--------------------------------------------------------------------------------------------
export DATA_FILE="/tmp/${HADOOP_NAME_IN_ZABBIX}_${MONITOR_TYPE}.txt"
export JSON_FILE="/tmp/${HADOOP_NAME_IN_ZABBIX}_${MONITOR_TYPE}.json"
export BAK_DATA_FILE="/tmp/${HADOOP_NAME_IN_ZABBIX}_${MONITOR_TYPE}_bak.txt"
export LOG_FILE="/tmp/${HADOOP_NAME_IN_ZABBIX}.log"


#--------------------------------------------------------------------------------------------
# Use python to get the metrics data from Hadoop NameNode and use screen-scraping to extract
# metrics. 
# The final result of screen scraping is a file containing data in the following format -
# <HADOOP_NAME_IN_ZABBIX> <METRIC_NAME> <METRIC_VALUE>
#--------------------------------------------------------------------------------------------

# python `dirname $0`/zabbix-hadoop.py $CLUSTER_HOST $METRICS_PORT $DATA_FILE $HADOOP_NAME_IN_ZABBIX $MONITOR_TYPE
wget -q -O $DATA_FILE http://$CLUSTER_HOST:$METRICS_PORT/jmx?qry=Hadoop:*

sh `dirname $0`/JSON.sh -l < $DATA_FILE > $JSON_FILE
#NN > JvmMetrics
heap_memory_used="`grep 'MemHeapUsedM' $JSON_FILE | sed -n '1,1p' | cut -f 2`"
heap_memory="`grep 'MemHeapCommittedM' $JSON_FILE | sed -n '1,1p' | cut -f 2`"
max_heap_memory="`grep 'MemHeapMaxM' $JSON_FILE | sed -n '1,1p' | cut -f 2`"
non_heap_memory_used="`grep 'MemNonHeapUsedM' $JSON_FILE | sed -n '1,1p' | cut -f 2`"
commited_non_heap_memory="`grep 'MemNonHeapCommittedM' $JSON_FILE | sed -n '1,1p' | cut -f 2`"
all_memory_used=`awk 'BEGIN{print $heap_memory_used + $non_heap_memory_used}'`
#NN > NameNodeStatus
namenode_state="`grep 'tag.HAState' $JSON_FILE | sed -n '1,1p' | cut -f 2`"
#NN > NameNodeInfo
start_time="`grep 'BlockDeletionStartTime' $JSON_FILE | sed -n '1,1p' | cut -f 2`"
hadoop_version=`grep 'SoftwareVersion' $JSON_FILE | sed -n '1,1p' | cut -f 2 | cut -d \" -f 2`
# DFS > FSNamesystemState
live_nodes="`grep 'NumLiveDataNodes' $JSON_FILE | sed -n '1,1p' | cut -f 2`" 
dead_nodes="`grep 'NumDeadDataNodes' $JSON_FILE | sed -n '1,1p' | cut -f 2`" 
decommissioning_nodes="`grep 'NumDecommissioningDataNodes' $JSON_FILE | sed -n '1,1p' | cut -f 2`"
under_replicated_blocks="`grep 'UnderReplicatedBlocks' $JSON_FILE | sed -n '1,1p' | cut -f 2`"
# DFS > NameNodeInfo
files_and_directorys="`grep 'TotalFiles' $JSON_FILE | sed -n '1,1p' | cut -f 2`"
blocks="`grep 'TotalBlocks' $JSON_FILE | sed -n '1,1p' | cut -f 2`"
configured_capacity="`grep '\"Total\"' $JSON_FILE | sed -n '1,1p' | cut -f 2`"
dfs_used="`grep 'CapacityUsed' $JSON_FILE | sed -n '1,1p' | cut -f 2`"
dfs_used_persent="`grep 'PercentUsed' $JSON_FILE | sed -n '1,1p' | cut -f 2`"
non_dfs_used="`grep 'NonDfsUsedSpace' $JSON_FILE | sed -n '1,1p' | cut -f 2`"
dfs_remaining="`grep 'Free' $JSON_FILE | sed -n '1,1p' | cut -f 2`"
dfs_remaining_persent="`grep 'PercentRemaining' $JSON_FILE | sed -n '1,1p' | cut -f 2`"
datanodes_usages="`grep 'NodeUsage' $JSON_FILE | sed -n '1,1p' | cut -f 2`"

echo "$HADOOP_NAME_IN_ZABBIX heap_memory_used $heap_memory_used
$HADOOP_NAME_IN_ZABBIX heap_memory $heap_memory
$HADOOP_NAME_IN_ZABBIX max_heap_memory $max_heap_memory
$HADOOP_NAME_IN_ZABBIX non_heap_memory_used $non_heap_memory_used
$HADOOP_NAME_IN_ZABBIX commited_non_heap_memory $commited_non_heap_memory
$HADOOP_NAME_IN_ZABBIX all_memory_used $all_memory_used
$HADOOP_NAME_IN_ZABBIX namenode_state $namenode_state
$HADOOP_NAME_IN_ZABBIX start_time $start_time
$HADOOP_NAME_IN_ZABBIX hadoop_version $hadoop_version
$HADOOP_NAME_IN_ZABBIX live_nodes $live_nodes
$HADOOP_NAME_IN_ZABBIX dead_nodes $dead_nodes
$HADOOP_NAME_IN_ZABBIX decommissioning_nodes $decommissioning_nodes
$HADOOP_NAME_IN_ZABBIX under_replicated_blocks $under_replicated_blocks
$HADOOP_NAME_IN_ZABBIX files_and_directorys $files_and_directorys
$HADOOP_NAME_IN_ZABBIX blocks $blocks
$HADOOP_NAME_IN_ZABBIX configured_capacity $configured_capacity
$HADOOP_NAME_IN_ZABBIX dfs_used $dfs_used
$HADOOP_NAME_IN_ZABBIX dfs_used_persent $dfs_used_persent
$HADOOP_NAME_IN_ZABBIX non_dfs_used $non_dfs_used
$HADOOP_NAME_IN_ZABBIX dfs_remaining $dfs_remaining
$HADOOP_NAME_IN_ZABBIX dfs_remaining_persent $dfs_remaining_persent
$HADOOP_NAME_IN_ZABBIX datanodes_usages $datanodes_usages" > $DATA_FILE



#--------------------------------------------------------------------------------------------
# Check the size of $DATA_FILE. If it is not empty, use zabbix_sender to send data to Zabbix.
#--------------------------------------------------------------------------------------------
if [[ -s $DATA_FILE ]]
then
   zabbix_sender -vv -z 127.0.0.1 -i $DATA_FILE 2>>$LOG_FILE 1>>$LOG_FILE
   echo  -e "Successfully executed $COMMAND_LINE" >>$LOG_FILE
   mv $DATA_FILE $BAK_DATA_FILE
   echo "OK"
else
   echo "Error in executing $COMMAND_LINE" >> $LOG_FILE
   echo "ERROR"
fi


# debug: sh cluster-hadoop-plugin.sh 10.41.236.209 50070 master1 DFS

cluster-hadoop-plugin.sh ${HADOOP_NAMENODE_HOST} ${HADOOP_NAMENODE_METRICS_PORT} ${ZABBIX_NAME} DFS

cat /etc/zabbix/zabbix_agentd.conf
### Option: EnableRemoteCommands
#       Whether remote commands from Zabbix server are allowed.
#       0 - not allowed
#       1 - allowed
#
# Mandatory: no
# Default:
# EnableRemoteCommands=0

cat config.yml
  - name: EnableRemoteCommands 
    lineinfile: 
      path: /etc/zabbix/zabbix_agentd.conf
      regexp: '(.*EnableRemoteCommands.*)'
      line: '#\1'
    become: yes

cat config.yml
  - name: EnableRemoteCommands 
    lineinfile: 
      path: /etc/zabbix/zabbix_agentd.conf
      regexp: '^#(.*EnableRemoteCommands.*)'
      line: '\1'
    become: yes