2013年8月8日星期四

設定 CDH4U3 Security for HFD/MR1

reference [CDH4U3 Security Guide]

CentOS 6

network

$ system-config-network

passwordless SSH

[root@0a90a489]

rm -rf /root/.ssh;mkdir /root/.ssh;chmod 700 /root/.ssh;ssh-keygen -P '' -f /root/.ssh/id_rsa;chmod 600 /root/.ssh/*

ls -ald /root/.ssh;ls -al /root/.ssh/*

[root@0a90a48a]

rm -rf /root/.ssh;mkdir /root/.ssh; chmod 700 /root/.ssh

scp master:/root/.ssh/id_rsa.pub /root/.ssh/authorized_keys

chmod 600 /root/.ssh/authorized_keys;restorecon -R -v /root/.ssh

ls -ald /root/.ssh;ls -al /root/.ssh/*

hosts file

[root@0a90a489] cat /etc/hosts

127.0.0.1 localhost

10.144.164.134 kdc.cht.local

10.144.164.137 0a90a489.cht.local 0a90a489

10.144.164.138 0a90a48a.cht.local 0a90a48a

10.144.164.139 0a90a48b.cht.local 0a90a48b

[root@0a90a489] vim scpfile.sh

hosts="0a90a4b0 0a90a4b1 0a90a4b2 0a90a4b3"

for i in `echo $hosts`

scp $1 $i:$1

ssh $i "ls -l $1"

done

NTP

setup ntp client on all hosts

[root@0a90a489] vim /etc/ntp.conf

driftfile /var/lib/ntp/drift

restrict default kod nomodify notrap nopeer noquery

restrict -6 default kod nomodify notrap nopeer noquery

restrict 127.0.0.1

restrict -6 ::1

restrict kdc.cht.local

server kdc.cht.local prefer

#server 0.centos.pool.ntp.org

#server 1.centos.pool.ntp.org

#server 2.centos.pool.ntp.org

includefile /etc/ntp/crypto/pw

keys /etc/ntp/keys

[root@0a90a489] service ntpd restart
ntpd (pid 2228) is running...
[root@0a90a489] chkconfig ntpd on

[root@0a90a489] while [ true ]; do clear; ntpstat; sleep 5; done

synchronised to NTP server (192.168.15.105) at stratum 12

time correct to within 955 ms

polling server every 64 s

[root@0a90a489] sh scpfile.sh /etc/ntp.conf
[root@0a90a489] ssh 0a90a48a "service ntpd restart"
[root@0a90a489] ssh 0a90a48a "chkconfig ntpd on"

Kerberos

setup Kerberos client on all hosts

[root@0a90a489] scp kdc.cht.local:/etc/krb5.conf /etc/krb5.conf
[root@0a90a489] vim /etc/krb5.conf

[logging]

default = FILE:/var/log/krb5libs.log

kdc = FILE:/var/log/krb5kdc.log

admin_server = FILE:/var/log/kadmind.log

[libdefaults]

default_realm = CHT.COM.TW

dns_lookup_realm = false

dns_lookup_kdc = false

ticket_lifetime = 24h

renew_lifetime = 1d

forwardable = true

[realms]

CHT.COM.TW = {

kdc = kdc.cht.local:88

admin_server = kdc.cht.local

default_domain = cht.local

}

[domain_realm]

.cht.local = CHT.COM.TW

cht.local = CHT.COM.TW

[root@0a90a489] yum install krb5-workstation

Installed:

krb5-workstation.x86_64 0:1.9-33.el6_3.3
Dependency Updated:

krb5-libs.x86_64 0:1.9-33.el6_3.3

[root@master] yum info krb5-workstation

Name : krb5-workstation

Arch : x86_64

Version : 1.9

Release : 33.el6_3.3

Size : 631 k

Repo : installed

[root@0a90a4ae] kinit leorick/admin
[root@0a90a4ae] klist -e
[root@0a90a4ae] kdestroy

[root@0a90a489] sh scpfile.sh /etc/krb5.conf

Step 3: If you are Using AES-256 Encryption, install the JCE Policy File on all hosts

[root@0a90a489] unzip jce_policy-6.zip

[root@0a90a489] cd $JAVA_HOME/jre/lib/security
[root@0a90a489] $ ls -l
-rw-r--r-- 1 root root 2481 May 8 15:15 local_policy.jar

-rw-r--r-- 1 root root 2465 May 8 15:15 US_export_policy.jar

[root@0a90a489] mv local_policy.jar local_policy.jar.bak[root@0a90a489] mv US_export_policy.jar US_export_policy.jar.bak[root@0a90a489] cp ~/jce/local_policy.jar ./[root@0a90a489] cp ~/jce/US_export_policy.jar ./

[root@0a90a489] ls -l
-rw-r--r-- 1 root root 2481 Jul 18 16:59 local_policy.jar

-rw-r--r-- 1 root root 2465 Jul 18 16:59 US_export_policy.jar

[root@0a90a489] sh scpfile.sh /usr/java/jdk1.6.0_33/jre/lib/security/local_policy.jar
[root@0a90a489] sh scpfile.sh /usr/java/jdk1.6.0_33/jre/lib/security/US_export_policy.jar

HDFS

Step 2: Verify User Accounts and Groups in CDH4 Due to Security

[root@0a90a4ae] ls -l /opt/hadoop/hdfs/dfs/name/

drwxr-xr-x 3 hdfs hdfs 4096 Jul 18 10:41 /opt/hadoop/hdfs/dfs/name/

[root@0a90a4ae] ssh slave1 ls -l /opt/hadoop/hdfs/dfs/data

drwx------ 3 hdfs hdfs 4096 Jul 18 10:41 /opt/hadoop/hdfs/dfs/data

[root@0a90a4ae] ls -ld /var/log/hadoop-hdfs

drwxrwxr-x 2 hdfs hadoop 4096 Jul 18 10:27 /var/log/hadoop-hdfs

[root@0a90a4ae] ssh slave1 ls -ld /var/log/hadoop-hdfs

drwxrwxr-x 2 hdfs hadoop 4096 Jul 18 10:27 /var/log/hadoop-hdfs

[root@0a90a4ae] ls -ld /opt/hadoop/mapred/local

drwxr-xr-x 3 mapred mapred 4096 Jul 18 10:27 /opt/hadoop/mapred/local

[root@0a90a4ae] ssh slave1 ls -ld /opt/hadoop/mapred/local

drwxr-xr-x 7 mapred mapred 4096 Jul 18 10:27 /opt/hadoop/mapred/local

[root@0a90a4ae] ls -ld /var/log/hadoop-0.20-mapreduce

drwxrwxr-x 3 root hadoop 4096 Jul 18 10:27 /var/log/hadoop-0.20-mapreduce

[root@0a90a4ae] ssh 0a90a48a ls -ld /var/log/hadoop-0.20-mapreduce

drwxrwxr-x 3 root hadoop 4096 Jul 18 10:27 /var/log/hadoop-0.20-mapreduce

Step 3: If you are Using AES-256 Encryption, install the JCE Policy File on all hosts

Step 4: Create and Deploy the Kerberos Principals and Keytab Files

[root@kdc] sh princ_hadoop_add.sh

domain="CHT.COM.TW"

instance="0a90a489.cht.local 0a90a48a.cht.local 0a90a48b.cht.local"

principals="HTTP hdfs mapred"

echo "you should authenticate with KDC first..."

/usr/kerberos/bin/klist

for i in `echo ${instance}`

echo "### HOST $i"

for j in `echo ${principals}`

/usr/sbin/kadmin.local -q "addprinc -randkey ${j}/${i}@${domain}"

done

/usr/sbin/kadmin.local -q "list_principals *"

HTTP/0a90a489.cht.local@CHT.COM.TW

HTTP/0a90a48a.cht.local@CHT.COM.TW

HTTP/0a90a48b.cht.local@CHT.COM.TW

hdfs/0a90a489.cht.local@CHT.COM.TW

hdfs/0a90a48a.cht.local@CHT.COM.TW

hdfs/0a90a48b.cht.local@CHT.COM.TW

mapred/0a90a489.cht.local@CHT.COM.TW

mapred/0a90a48a.cht.local@CHT.COM.TW

mapred/0a90a48b.cht.local@CHT.COM.TW

[root@kdc] sh princ_hadoop_del.sh

domain="CHT.COM.TW"

instance="0a90a489.cht.local 0a90a48a.cht.local 0a90a48b.cht.local"

principals="HTTP hdfs mapred"

echo "you should authenticate with KDC first..."

/usr/kerberos/bin/klist

for i in `echo ${instance}`

echo "### HOST $i"

for j in `echo ${principals}`

/usr/sbin/kadmin.local -q "delprinc -force ${j}/${i}@${domain}"

done

/usr/sbin/kadmin.local -q "list_principals *"

[root@0a90a489] kdestroy; kinit leorick/admin@CHT.COM.TW
[root@0a90a489] sh key_hadoop_dist.sh

domain="CHT.COM.TW"

instance="0a90a489.cht.local 0a90a48a.cht.local 0a90a48b.cht.local"

echo "you should authenticate with KDC first..."

/usr/bin/klist

for i in `echo ${instance}`

echo "### HOST $i"

KEYTAB="hadoop.keytab"

TMPFILE="/root/${KEYTAB}.${i}"

DESFILE="/etc/hadoop/conf/${KEYTAB}"

PRINCIPAL="HTTP/${i}@${domain} mapred/${i}@${domain} hdfs/${i}@${domain}"

rm -f ${TMPFILE}

/usr/bin/kadmin -q "ktadd -k ${TMPFILE} ${PRINCIPAL}"

scp ${TMPFILE} ${i}:${DESFILE}

CMD="chown hdfs:hadoop ${DESFILE}"

ssh ${i} ${CMD}

CMD="chmod 440 ${DESFILE}"

ssh ${i} ${CMD}

CMD="ls -l ${DESFILE}"

ssh ${i} ${CMD}

done

[root@0a90a489] vim key_hadoop_list.sh

target=`ls /etc/hadoop/conf/*.keytab`

for i in ${target}

echo `ls -l ${i}`

sudo /usr/bin/ktutil << EOF

rkt ${i}

list -e

EOF

done

[root@0a90a489] sh scpfile.sh /root/key_hadoop_list.sh

[root@0a90a489] sh key_hadoop_list.sh

-r--r----- 1 hdfs hadoop 1352 Jul 31 15:23 /etc/hadoop/conf/hadoop.keytab

ktutil: rkt /etc/hadoop/conf/hadoop.keytab

ktutil: list -e

slot KVNO Principal

---- ---- ---------------------------------------------------------------------

1 2 HTTP/0a90a489.cht.local@CHT.COM.TW (aes256-cts-hmac-sha1-96)

2 2 HTTP/0a90a489.cht.local@CHT.COM.TW (aes128-cts-hmac-sha1-96)

3 2 HTTP/0a90a489.cht.local@CHT.COM.TW (des3-cbc-sha1)

4 2 HTTP/0a90a489.cht.local@CHT.COM.TW (arcfour-hmac)

5 2 HTTP/0a90a489.cht.local@CHT.COM.TW (des-hmac-sha1)

6 2 HTTP/0a90a489.cht.local@CHT.COM.TW (des-cbc-md5)

7 2 mapred/0a90a489.cht.local@CHT.COM.TW (aes256-cts-hmac-sha1-96)

8 2 mapred/0a90a489.cht.local@CHT.COM.TW (aes128-cts-hmac-sha1-96)

9 2 mapred/0a90a489.cht.local@CHT.COM.TW (des3-cbc-sha1)

10 2 mapred/0a90a489.cht.local@CHT.COM.TW (arcfour-hmac)

11 2 mapred/0a90a489.cht.local@CHT.COM.TW (des-hmac-sha1)

12 2 mapred/0a90a489.cht.local@CHT.COM.TW (des-cbc-md5)

13 2 hdfs/0a90a489.cht.local@CHT.COM.TW (aes256-cts-hmac-sha1-96)

14 2 hdfs/0a90a489.cht.local@CHT.COM.TW (aes128-cts-hmac-sha1-96)

15 2 hdfs/0a90a489.cht.local@CHT.COM.TW (des3-cbc-sha1)

16 2 hdfs/0a90a489.cht.local@CHT.COM.TW (arcfour-hmac)

17 2 hdfs/0a90a489.cht.local@CHT.COM.TW (des-hmac-sha1)

18 2 hdfs/0a90a489.cht.local@CHT.COM.TW (des-cbc-md5)

[root@0a90a489] ssh 0a90a48a "sh /root/key_hadoop_list.sh"

-r--r----- 1 hdfs hadoop 1352 Jul 31 15:22 /etc/hadoop/conf/hadoop.keytab

ktutil: rkt /etc/hadoop/conf/hadoop.keytab

ktutil: list -e

slot KVNO Principal

---- ---- ---------------------------------------------------------------------

1 2 HTTP/0a90a48a.cht.local@CHT.COM.TW (aes256-cts-hmac-sha1-96)

2 2 HTTP/0a90a48a.cht.local@CHT.COM.TW (aes128-cts-hmac-sha1-96)

3 2 HTTP/0a90a48a.cht.local@CHT.COM.TW (des3-cbc-sha1)

4 2 HTTP/0a90a48a.cht.local@CHT.COM.TW (arcfour-hmac)

5 2 HTTP/0a90a48a.cht.local@CHT.COM.TW (des-hmac-sha1)

6 2 HTTP/0a90a48a.cht.local@CHT.COM.TW (des-cbc-md5)

7 2 mapred/0a90a48a.cht.local@CHT.COM.TW (aes256-cts-hmac-sha1-96)

8 2 mapred/0a90a48a.cht.local@CHT.COM.TW (aes128-cts-hmac-sha1-96)

9 2 mapred/0a90a48a.cht.local@CHT.COM.TW (des3-cbc-sha1)

10 2 mapred/0a90a48a.cht.local@CHT.COM.TW (arcfour-hmac)

11 2 mapred/0a90a48a.cht.local@CHT.COM.TW (des-hmac-sha1)

12 2 mapred/0a90a48a.cht.local@CHT.COM.TW (des-cbc-md5)

13 2 hdfs/0a90a48a.cht.local@CHT.COM.TW (aes256-cts-hmac-sha1-96)

14 2 hdfs/0a90a48a.cht.local@CHT.COM.TW (aes128-cts-hmac-sha1-96)

15 2 hdfs/0a90a48a.cht.local@CHT.COM.TW (des3-cbc-sha1)

16 2 hdfs/0a90a48a.cht.local@CHT.COM.TW (arcfour-hmac)

17 2 hdfs/0a90a48a.cht.local@CHT.COM.TW (des-hmac-sha1)

18 2 hdfs/0a90a48a.cht.local@CHT.COM.TW (des-cbc-md5)

[root@0a90a489] ssh 0a90a48b "sh /root/key_hadoop_list.sh"

-r--r----- 1 hdfs hadoop 1352 Jul 31 15:30 /etc/hadoop/conf/hadoop.keytab

ktutil: rkt /etc/hadoop/conf/hadoop.keytab

ktutil: list -e

slot KVNO Principal

---- ---- ---------------------------------------------------------------------

1 2 HTTP/0a90a48b.cht.local@CHT.COM.TW (aes256-cts-hmac-sha1-96)

2 2 HTTP/0a90a48b.cht.local@CHT.COM.TW (aes128-cts-hmac-sha1-96)

3 2 HTTP/0a90a48b.cht.local@CHT.COM.TW (des3-cbc-sha1)

4 2 HTTP/0a90a48b.cht.local@CHT.COM.TW (arcfour-hmac)

5 2 HTTP/0a90a48b.cht.local@CHT.COM.TW (des-hmac-sha1)

6 2 HTTP/0a90a48b.cht.local@CHT.COM.TW (des-cbc-md5)

7 2 mapred/0a90a48b.cht.local@CHT.COM.TW (aes256-cts-hmac-sha1-96)

8 2 mapred/0a90a48b.cht.local@CHT.COM.TW (aes128-cts-hmac-sha1-96)

9 2 mapred/0a90a48b.cht.local@CHT.COM.TW (des3-cbc-sha1)

10 2 mapred/0a90a48b.cht.local@CHT.COM.TW (arcfour-hmac)

11 2 mapred/0a90a48b.cht.local@CHT.COM.TW (des-hmac-sha1)

12 2 mapred/0a90a48b.cht.local@CHT.COM.TW (des-cbc-md5)

13 2 hdfs/0a90a48b.cht.local@CHT.COM.TW (aes256-cts-hmac-sha1-96)

14 2 hdfs/0a90a48b.cht.local@CHT.COM.TW (aes128-cts-hmac-sha1-96)

15 2 hdfs/0a90a48b.cht.local@CHT.COM.TW (des3-cbc-sha1)

16 2 hdfs/0a90a48b.cht.local@CHT.COM.TW (arcfour-hmac)

17 2 hdfs/0a90a48b.cht.local@CHT.COM.TW (des-hmac-sha1)

18 2 hdfs/0a90a48b.cht.local@CHT.COM.TW (des-cbc-md5)

Step 5: Shut Down the Cluster

service hbase-regionserver stop
service zookeeper-server stop
service hbase-master stop
for x in `cd /etc/init.d ; ls hadoop-*` ; do sudo service $x stop ; done
service hbasestargate stop
service oozie stop
service hue stop

Step 6.1: Enable Hadoop Security

[root@0a90a489] vim /etc/hadoop/conf/core-site.xml

hadoop.security.authentication
kerberos

hadoop.security.authorization
true

[root@0a90a489] sh scpfile.sh /etc/hadoop/conf/core-site.xml

Step 7: Configure Secure HDFS

[root@0a90a489] vim /etc/hadoop/conf/hdfs-site.xml

dfs.block.access.token.enable
true

dfs.namenode.keytab.file
/etc/hadoop/conf/hadoop.keytab

dfs.namenode.kerberos.principal
hdfs/_HOST@CHT.COM.TW

dfs.namenode.kerberos.internal.spnego.principal
HTTP/_HOST@CHT.COM.TW

dfs.secondary.namenode.keytab.file
/etc/hadoop/conf/hadoop.keytab

dfs.secondary.namenode.kerberos.principal
hdfs/_HOST@CHT.COM.TW

dfs.secondary.namenode.kerberos.internal.spnego.principal
HTTP/_HOST@CHT.COM.TW

dfs.datanode.data.dir.perm
700

dfs.datanode.address
0.0.0.0:1004

dfs.datanode.http.address
0.0.0.0:1006

dfs.datanode.keytab.file
/etc/hadoop/conf/hadoop.keytab

dfs.datanode.kerberos.principal
hdfs/_HOST@CHT.COM.TW

[root@0a90a489] sh scpfile.sh /etc/hadoop/conf/hdfs-site.xml

Step 10: Set Variables for Secure DataNodes

[root@0a90a489] vim /etc/default/hadoop-hdfs-datanode

export HADOOP_SECURE_DN_USER=hdfs
export HADOOP_SECURE_DN_PID_DIR=/var/run/hadoop-hdfs

export HADOOP_SECURE_DN_LOG_DIR=/var/log/hadoop-hdfs

export JSVC_HOME=/usr/libexec/bigtop-utils

[root@0a90a489] sh scpfile.sh /etc/default/hadoop-hdfs-datanode

Step 11.4: Start up the NameNode

[root@0a90a489] service hadoop-hdfs-namenode start
[root@0a90a489] vim /var/log/hadoop-hdfs/hadoop-hdfs-namenode-0a90a489.log

...

2013-07-31 16:15:54,732 INFO org.apache.hadoop.security.UserGroupInformation: Login successful for user hdfs/0a90a489.cht.local@CHT.COM.TW using keytab file /etc/hadoop/conf/hadoop.keytab

...

2013-07-31 16:15:56,615 INFO org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler: Login using keytab /etc/hadoop/conf/hadoop.keytab, for principal HTTP/0a90a489.cht.local@CHT.COM.TW

2013-07-31 16:15:56,634 INFO org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler: Initialized, principal [HTTP/0a90a489.cht.local@CHT.COM.TW] from keytab [/etc/hadoop/conf/hadoop.keytab]

[root@0a90a489] kdestroy
[root@0a90a489] hadoop fs -ls /
13/07/31 16:20:34 ERROR security.UserGroupInformation: PriviledgedActionException as:root (auth:KERBEROS) cause:javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)]

[root@0a90a489] kinit leorick/admin

Password for leorick/admin@CHT.COM.TW:

[root@0a90a489] hadoop fs -ls /

Found 6 items

drwxr-xr-x - mapred hadoop 0 2013-07-31 14:27 /benchmarks

drwxr-xr-x - hbase hbase 0 2013-07-29 16:58 /hbase

drwxr-xr-x - mapred mapred 0 2013-07-29 16:26 /mapred

drwxrwxrwt - hdfs hadoop 0 2013-07-31 14:06 /tmp

drwxr-xr-x - hdfs hadoop 0 2013-07-29 16:24 /user

drwxr-xr-x - hdfs hadoop 0 2013-07-29 16:25 /var

Step 12.1: Start up a DataNode

[root@0a90a48a] service hadoop-hdfs-datanode start
[root@0a90a48a] vim /var/log/hadoop-hdfs/hadoop-hdfs-datanode-0a90a48a.log

2013-07-31 16:22:47,662 INFO org.apache.hadoop.security.UserGroupInformation: Login successful for user hdfs/0a90a48a.cht.local@CHT.COM.TW using keytab file /etc/hadoop/conf/hadoop.keytab

[root@0a90a48b ] sudo service hadoop-hdfs-datanode start
[root@0a90a48b ] vim /var/log/hadoop-hdfs/hadoop-hdfs-datanode-0a90a48b.log

2013-07-31 16:32:26,031 INFO org.apache.hadoop.security.UserGroupInformation: Login successful for user hdfs/0a90a48b.cht.local@CHT.COM.TW using keytab file /etc/hadoop/conf/hadoop.keytab

Step 12.2: Test

[root@0a90a489] kdestroy;kinit -k -t /etc/hadoop/conf/hadoop.keytab hdfs/0a90a489.cht.local@CHT.COM.TW

[root@0a90a489] hdfs dfs -mkdir /user/leorick

[root@0a90a489] hdfs dfs -put install.log /user/leorick/

[root@0a90a489] hdfs dfs -chown -R leorick:leorick /user/leorick

[root@0a90a489] hdfs dfs -chmod -R 700 /user/leorick

[root@0a90a489] hdfs dfs -ls -R /user/leorick

-rw------- 2 leorick leorick 25233 2013-07-31 16:36 /user/leorick/install.log

[root@0a90a489] hdfs dfs -cat /user/leorick/install.log

Installing libgcc-4.4.7-3.el6.x86_64

warning: libgcc-4.4.7-3.el6.x86_64: Header V3 RSA/SHA1 Signature, key ID c105b9de: NOKEY
...

[root@0a90a489] kdestroy;kinit leorick/admin@CHT.COM.TW

[root@0a90a489] hdfs dfs -cat /user/leorick/install.log

Installing libgcc-4.4.7-3.el6.x86_64

warning: libgcc-4.4.7-3.el6.x86_64: Header V3 RSA/SHA1 Signature, key ID c105b9de: NOKEY
...

Step 13: Set the Sticky Bit on HDFS Directories

[root@0a90a489] sudo -u hdfs kinit -k -t /etc/hadoop/conf/hadoop.keytab hdfs/0a90a489.cht.local@CHT.COM.TW

[root@0a90a489] sudo -u hdfs hadoop fs -chmod 1777 /tmp

[root@0a90a489] sudo -u hdfs hadoop fs -ls /

drwxrwxrwt - hdfs hadoop 0 2013-07-31 14:06 /tmp

drwxr-xr-x - hdfs hadoop 0 2013-07-31 16:34 /user

MapReduce 0.20

Step 1: Configure Secure MRv1

[root@0a90a489] vim /etc/hadoop/conf/mapred-site.xml

mapreduce.jobtracker.kerberos.principal

mapred/_HOST@CHT.COM.TW

mapreduce.jobtracker.keytab.file

/etc/hadoop/conf/hadoop.keytab

mapreduce.tasktracker.kerberos.principal

mapred/_HOST@CHT.COM.TW

mapreduce.tasktracker.keytab.file

/etc/hadoop/conf/hadoop.keytab

mapred.task.tracker.task-controller

org.apache.hadoop.mapred.LinuxTaskController

mapreduce.tasktracker.group

mapred

[root@0a90a489] vim /etc/hadoop/conf/taskcontroller.cfg

hadoop.log.dir=/var/log/hadoop-0.20-mapreduce/

mapreduce.tasktracker.group=mapred

banned.users=bin

min.user.id=100

[root@0a90a489] sh scpfile.sh /etc/hadoop/conf/mapred-site.xml

[root@0a90a489] sh scpfile.sh /etc/hadoop/conf/taskcontroller.cfg

Step 2: Start up the JobTracker

[root@0a90a489] service hadoop-0.20-mapreduce-jobtracker start

[root@0a90a489] vim /var/log/hadoop-0.20-mapreduce/hadoop-hadoop-jobtracker-0a90a489.log

2013-07-31 17:03:31,296 INFO org.apache.hadoop.security.UserGroupInformation: Login successful for user mapred/0a90a489.cht.local@CHT.COM.TW using keytab file /etc/hadoop/conf/hadoop.keytab

Step 3.1: Start up a TaskTracker

[root@0a90a48a] service hadoop-0.20-mapreduce-tasktracker start

[root@0a90a48a] vim /var/log/hadoop-0.20-mapreduce/hadoop-hadoop-tasktracker-0a90a48a.log

2013-07-31 17:12:38,879 INFO org.apache.hadoop.security.UserGroupInformation: Login successful for user mapred/0a90a48a.cht.local@CHT.COM.TW using keytab file /etc/hadoop/conf/hadoop.keytab

[root@0a90a48b] service hadoop-0.20-mapreduce-tasktracker start

[root@0a90a48b] vim /var/log/hadoop-0.20-mapreduce/hadoop-hadoop-tasktracker-0a90a48b.log

2013-07-31 17:11:15,282 INFO org.apache.hadoop.security.UserGroupInformation: Login successful for user mapred/0a90a48b.cht.local@CHT.COM.TW using keytab file /etc/hadoop/conf/hadoop.keytab

Step 3.2: Testing

3.2.1 沒有kerberos principle執行程式

[root@0a90a489] kdestroy
[root@0a90a489] hadoop jar /usr/lib/hadoop-0.20-mapreduce/hadoop-examples.jar pi 4 1000

Number of Maps = 4

Samples per Map = 1000

13/07/31 17:14:26 ERROR security.UserGroupInformation: PriviledgedActionException as:root (auth:KERBEROS) cause:javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)]

3.2.2 在每台主機建立OS帳號

[root@0a90a489] cat /etc/group

hadoop:x:498:

[root@0a90a489] useradd -b /home -k /etc/skel -m -s /bin/bash -g 498 leorick

[root@0a90a489] ssh 0a90a48a "useradd -b /home -k /etc/skel -m -s /bin/bash -g 498 leorick"

[root@0a90a489] ssh 0a90a48b "useradd -b /home -k /etc/skel -m -s /bin/bash -g 498 leorick"

[root@0a90a489] id leorick

uid=500(leorick) gid=498(hadoop) groups=498(hadoop)

[root@0a90a489] ssh 0a90a48a "id leorick"

uid=500(leorick) gid=498(hadoop) groups=498(hadoop)

[root@0a90a489] ssh 0a90a48b "id leorick"

uid=500(leorick) gid=498(hadoop) groups=498(hadoop)

3.2.3 使用kerberos principle執行程式

[root@0a90a489] kinit leorick/admin

Password for leorick/admin@CHT.COM.TW:

[root@0a90a489] hadoop jar /usr/lib/hadoop-0.20-mapreduce/hadoop-examples.jar pi 4 1000

…

13/07/31 17:15:27 ERROR security.UserGroupInformation: PriviledgedActionException as:leorick/admin@CHT.COM.TW (auth:KERBEROS) cause:org.apache.hadoop.security.AccessControlException: Permission denied: user=leorick, access=EXECUTE, inode="/tmp/hadoop-mapred":mapred:hadoop:drwx------

...

3.2.4 修改HDFS上共用目錄權限並執行程式

[root@0a90a48a] kdestroy;kinit -k -t /etc/hadoop/conf/hadoop.keytab hdfs/0a90a48a.cht.local@CHT.COM.TW

[root@0a90a48a] hdfs dfs -chmod -R 777 /tmp/hadoop-mapred

[root@0a90a489] hadoop jar /usr/lib/hadoop-0.20-mapreduce/hadoop-examples.jar pi 4 1000

Starting Job

13/07/31 17:28:40 WARN mapred.JobClient: Use GenericOptionsParser for parsing the arguments. Applications should implement Tool for the same.

13/07/31 17:28:40 INFO hdfs.DFSClient: Created HDFS_DELEGATION_TOKEN token 2 for leorick on 10.144.164.137:8020

13/07/31 17:28:40 INFO security.TokenCache: Got dt for hdfs://0a90a489.cht.local; Kind: HDFS_DELEGATION_TOKEN, Service: 10.144.164.137:8020, Ident: (HDFS_DELEGATION_TOKEN token 2 for leorick)

…

Job Finished in 42.958 seconds

Estimated value of Pi is 3.14000000000000000000

2013年7月17日星期三

設定 CDH3U5 Audit Log

複製/usr/lib/hadoop/example-confs/conf.secure/log4j.properties到/etc/hadoop/conf/log4j.properties 並修改內容

# Define some default values that can be overridden by system properties
hadoop.root.logger=INFO,console
hadoop.log.dir=.
hadoop.log.file=hadoop.log

# Define the root logger to the system property "hadoop.root.logger".
log4j.rootLogger=${hadoop.root.logger}, EventCounter

# Logging Threshold
log4j.threshhold=ALL

#
# Daily Rolling File Appender
#

log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}
log4j.appender.DRFA.DatePattern=.yyyy-MM-dd

# 30-day backup
#log4j.appender.DRFA.MaxBackupIndex=30
log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout

# Pattern format: Date LogLevel LoggerName LogMessage
log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
# Debugging Pattern format
#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n

#
# console
# Add "console" to rootlogger above if you want to use this
#

log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.target=System.err
log4j.appender.console.layout=org.apache.log4j.PatternLayout
log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n

#
# TaskLog Appender
#

#Default values
hadoop.tasklog.taskid=null
hadoop.tasklog.noKeepSplits=4
hadoop.tasklog.totalLogFileSize=100
hadoop.tasklog.purgeLogSplits=true
hadoop.tasklog.logsRetainHours=12
hadoop.tasklog.iscleanup=false

log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender
log4j.appender.TLA.taskId=${hadoop.tasklog.taskid}
log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}

log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}

log4j.appender.TLA.layout=org.apache.log4j.PatternLayout
log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n

#
# Event Counter Appender
# Sends counts of logging messages at different severity levels to Hadoop Metrics.
#
log4j.appender.EventCounter=org.apache.hadoop.metrics.jvm.EventCounter

#=======
# security audit logging

security.audit.logger=INFO,DRFAS
log4j.category.SecurityLogger=${security.audit.logger}
log4j.additivity.SecurityLogger=false
hadoop.security.log.file=SecurityAuth-${user.name}.audit
log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender
log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.id.str}-auth.log
log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout
log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd

# hdfs audit logging

hdfs.audit.logger=INFO,DRFAAUDIT
log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}
log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false
log4j.appender.DRFAAUDIT=org.apache.log4j.DailyRollingFileAppender
log4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log
log4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout
log4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
log4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd

# mapred audit logging

mapred.audit.logger=INFO,MRAUDIT
log4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}
log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false
log4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender
log4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log
log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout
log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
log4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd

# Mapred job summary

mapred.jobsummary.logger=INFO,console
log4j.logger.org.apache.hadoop.mapred.JobInProgress$JobSummary=${mapred.jobsummary.logger}
log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary=false
log4j.appender.JSA=org.apache.log4j.DailyRollingFileAppender
log4j.appender.JSA.File=${hadoop.log.dir}/mapred-jobsummary.log
log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
log4j.appender.JSA.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
log4j.appender.JSA.DatePattern=.yyyy-MM-dd

在/var/log/hadoop目錄下產生2個audit files

-rw-r--r-- 1 hdfs hdfs 0 Jul 15 15:12 /var/log/hadoop/hdfs-audit.log
-rw-r--r-- 1 hdfs hdfs 0 Jul 15 15:12 /var/log/hadoop/mapred-audit.log

在另一台client執行hdfs指令並檢視audit log內容

[leo@slave1]# hadoop fs -ls /user/leo
drwxr-xr-x - leo leo 0 2013-05-30 14:34 /user/leo/warehouse
[leo@slave1]# hadoop fs -put boot.log /user/leo
[leo@slave1]# hadoop fs -chmod 777 /user/leo/boot.log
[leo@slave1]# hadoop fs -cat /user/leo/boot.log
[leo@slave1]# hadoop fs -rm /user/leo/boot.log

[root@master]# vim /var/log/hadoop/hdfs-audit.log
2013-07-15 15:14:49,637 INFO FSNamesystem.audit: ugi=leo (auth:SIMPLE) ip=/10.144.129.138 cmd=listStatus src=/user/leo dst=null perm=null
2013-07-15 15:20:35,159 INFO FSNamesystem.audit: ugi=leo (auth:SIMPLE) ip=/10.144.129.138 cmd=create src=/user/leo/boot.log dst=null perm=leo:leo:rw-r--r--
2013-07-15 15:21:13,695 INFO FSNamesystem.audit: ugi=leo (auth:SIMPLE) ip=/10.144.129.138 cmd=setPermission src=/user/leo/boot.log dst=null perm=leo:leo:rw-rw-rw-
2013-07-15 15:21:27,975 INFO FSNamesystem.audit: ugi=leo (auth:SIMPLE) ip=/10.144.129.138 cmd=open src=/user/leo/boot.log dst=null perm=null
2013-07-15 15:21:54,751 INFO FSNamesystem.audit: ugi=leo (auth:SIMPLE) ip=/10.144.129.138 cmd=delete src=/user/leo/boot.log dst=null perm=null

在另一台client執行hadoop job指令並檢視audit log內容

[leo@slave1]# hadoop jar /usr/lib/hadoop/hadoop-examples.jar pi 6 50000

[root@master]# ls /var/log/hadoop/mapred-auth.log
-rw-r--r-- 1 mapred mapred 0 Jul 15 15:25 /var/log/hadoop/mapred-auth.log

由於master上mapreduce audit log沒有內容, 因此修改hdfs & mapreduce audit file屬性, 並在client執行hadoop job

[root@master]# chown hdfs:hadoop /var/log/hadoop/hdfs-audit.log
[root@master]# chmod 774 /var/log/hadoop/hdfs-audit.log
[root@master]# chown hdfs:hadoop /var/log/hadoop/mapred-audit.log
[root@master]# chmod 774 /var/log/hadoop/mapred-audit.log

[leo@slave1]# hadoop jar /usr/lib/hadoop/hadoop-examples.jar pi 6 50000

hadoop job -status job_201307151623_0007

hadoop job -set-priority job_201307151623_0007 LOW

hadoop job -kill job_201307151623_0007

[root@master]# ls -l /var/log/hadoop/*au*
-rwxrwxr-- 1 hdfs hadoop 209864 Jul 15 16:33 /var/log/hadoop/hdfs-audit.log
-rwxrwxr-- 1 hdfs hadoop 987 Jul 15 16:33 /var/log/hadoop/mapred-audit.log
[root@master]# vim /var/log/hadoop/mapred-audit.log
2013-07-15 16:33:25,875 INFO mapred.AuditLogger: USER=leo IP=10.144.129.138 OPERATION=SUBMIT_JOB TARGET=job_201307151623_0007 RESULT=SUCCESS
#其餘指令皆無log

雖然手動修改log屬性, 然而daily rotate時又會恢復為預設值, 導致mapreduce audit log無法繼續寫入

2013年7月5日星期五

Big Data on Cloud

Big Data on Cloud這個概念所強調的並不是在於執行效能能夠更好(虛擬機怎麼會贏過實體機?!), 而是透過抽象化來提供更簡化,彈性的應用

這個概念在過去10多年來可以看到:

OOP (功能與屬性的抽象化封裝為物件, 功能相依性抽像化為繼承)
Java的JVM與.NET的CLR (程式執行環境的抽象化)
SOA (將程式API介面抽像化為應用服務訊息介接)

最近看到一則關於Cassandra應用的報導, 其中讓我感觸最深的就是開發商在短時間內就能夠將Cassandra叢集節點增加(6台->24台), 以應付突然增加的流量; 然後再進行問題確認與效能優化調校; 若是採用實體機佈署的話, 恐怕時間不知道要花上多久 (當然強而有力且選擇性豐富的AWS Infrastructure也是非常重要的關鍵)

2013/01 (iThome) 力可科技導入Cassandra－進軍全球市場的關鍵利器，10分鐘能擴充20萬人用量

2013年5月28日星期二

Setup Stargate for Apache HBase

Reference

HBase Wiki: Stargate

Experiments with the HBase REST API
JSONLint: The JSON Validator

Deployment

start daemon as root (port 51111 by BDCMS)
$ ./bin/hbase-daemon.sh start rest -p

check process
$ ps -ef|grep org.apache.hadoop.hbase.rest.Main

Query

Query software version

$ curl -s -i http://0a908189.cht.local:51111/version

HTTP/1.1 200 OK
Content-Length: 135
Cache-Control: no-cache
Content-Type: text/plain

rest 0.0.2 [JVM: Sun Microsystems Inc. 1.6.0_33-20.8-b03] [OS: Linux 2.6.32-358.el6.x86_64 amd64] [Server: jetty/6.1.26] [Jersey: 1.8]

$ curl -s -i -H "Accept: text/xml" 0a908189.cht.local:51111/version

HTTP/1.1 200 OK

Cache-Control: no-cache

Content-Type: text/xml

Content-Length: 198

$ curl -s -i -H "Accept: application/json" 0a908189.cht.local:51111/version

HTTP/1.1 200 OK

Cache-Control: no-cache

Content-Type: application/json

Transfer-Encoding: chunked

{"JVM":"Sun Microsystems Inc. 1.6.0_33-20.8-b03","Jersey":"1.8","OS":"Linux 2.6.32-358.el6.x86_64 amd64","REST":"0.0.2","Server":"jetty/6.1.26"}

Query Storage Cluster Version

$ curl -s -i -H "Accept: application/json" 0a908189.cht.local:51111/version/cluster

HTTP/1.1 200 OK

Cache-Control: no-cache

Content-Type: application/json

Transfer-Encoding: chunked

"0.90.6-cdh3u5"

Query Table List

$ curl -s -i -H "Accept: application/json" 0a908189.cht.local:51111

HTTP/1.1 200 OK

Cache-Control: no-cache

Content-Type: application/json

Transfer-Encoding: chunked

{"table":[{"name":"t1"}]}

Query Table Schema

$ curl -s -i -H "Accept: application/json" 0a908189.cht.local:51111/t1/schema

HTTP/1.1 200 OK

Cache-Control: no-cache

Content-Type: application/json

Transfer-Encoding: chunked

{"name":"t1","IS_META":"false","IS_ROOT":"false","ColumnSchema":[{"name":"fam1","BLOCKSIZE":"65536","BLOOMFILTER":"NONE","BLOCKCACHE":"true","COMPRESSION":"NONE","VERSIONS":"3","REPLICATION_SCOPE":"0","TTL":"2147483647","IN_MEMORY":"false"}]}

Cell Query (Single Value)

$ curl -s -i -H "Accept: application/json" 0a908189.cht.local:51111/t1/rlog-1369657973498-0a908185/fam1:host

HTTP/1.1 200 OK
Content-Type: application/json
Transfer-Encoding: chunked

{"Row":[{"key":"cmxvZy0xMzY5NjU3OTczNDk4LTBhOTA4MTg1","Cell":[{"column":"ZmFtMTpob3N0","timestamp":1369657977691,"$":"MGE5MDgxODU="}]}]}

$ CN=$(curl -s -H "Accept: application/json" 0a908189.cht.local:51111/t1/rlog-1369657973498-0a908185/fam1:host | grep -Po '"column":.*?[^\\]",' | tr -d '",'|awk -F: '{ print $2 }'|base64 -d);echo $CN

fam1:host

$ TM=$(curl -s -H "Accept: application/json" 0a908189.cht.local:51111/t1/rlog-1369657973498-0a908185/fam1:host | grep -Po '"timestamp":.*?[^\\],' |tr -d '",'|awk -F: '{ print $2 }');echo $TM

1369657977691

$ VL=$(curl -s -H "Accept: application/json" 0a908189.cht.local:51111/t1/rlog-1369657973498-0a908185/fam1:host | grep -Po '"\$":.*?[^\\]"' |tr -d '"'|awk -F: '{ print $2 }'|base64 -d);echo $VL

0a908185

$ VL=$(curl -s -H "Accept: application/json" 0a908189.cht.local:51111/t1/rlog-1369657973498-0a908185/fam1:body | grep -Po '"\$":.*?[^\\]"' |tr -d '"'|awk -F: '{ print $2 }'|base64 -d);echo $VL

sshd[3369]: pam_unix(sshd:session): session closed for user root

$ echo "ZmFtMTpGYWNpbGl0eQ=="|base64 -d
fam1:Facility
$ echo "ZmFtMTpTZXZlcml0eQ=="|base64 -d
fam1:Severity
$ echo "ZmFtMTphZ2VudA=="|base64 -d
fam1:agent
$ echo "ZmFtMTpib2R5"|base64 -d
fam1:body
echo "ZmFtMTpob3N0"|base64 -d
fam1:host
echo "ZmFtMTp0aW1lc3RhbXA="|base64 -d
fam1:timestamp
$ curl -s -H "Accept: application/json" 0a908189.cht.local:51111/t1/rlog-13697257366*

{
"Row": [
{
"key": "cmxvZy0xMzY5NzI1NzM2NjU5LTBhOTA4MThh",
"Cell": [
{
"column": "ZmFtMTpGYWNpbGl0eQ==",
"timestamp": 1369725741468,
"$": "ODA="
},
{
"column": "ZmFtMTpTZXZlcml0eQ==",
"timestamp": 1369725741467,
"$": "Ng=="
},
{
"column": "ZmFtMTphZ2VudA==",
"timestamp": 1369725741470,
"$": "MTAuMTQ0LjEyOS4xMzc="
},
{
"column": "ZmFtMTpib2R5",
"timestamp": 1369725741472,
"$": "c3NoZFsxMjg2OF06IEFjY2VwdGVkIHB1YmxpY2tleSBmb3Igcm9vdCBmcm9tIDEwLjE0NC4xMjkuMTM3IHBvcnQgNDY1MDcgc3NoMg=="
},
{
"column": "ZmFtMTpob3N0",
"timestamp": 1369725741469,
"$": "MGE5MDgxOGE="
},
{
"column": "ZmFtMTp0aW1lc3RhbXA=",
"timestamp": 1369725741466,
"$": "MTM2OTcyNTczNjY1OQ=="
}
]
},
{
"key": "cmxvZy0xMzY5NzI1NzM2NjYxLTBhOTA4MThh",
"Cell": [
{
"column": "ZmFtMTpGYWNpbGl0eQ==",
"timestamp": 1369725741474,
"$": "ODA="
},
{
"column": "ZmFtMTpTZXZlcml0eQ==",
"timestamp": 1369725741473,
"$": "Ng=="
},
{
"column": "ZmFtMTphZ2VudA==",
"timestamp": 1369725741475,
"$": "MTAuMTQ0LjEyOS4xMzc="
},
{
"column": "ZmFtMTpib2R5",
"timestamp": 1369725741476,
"$": "c3NoZFsxMjg2OF06IHBhbV91bml4KHNzaGQ6c2Vzc2lvbik6IHNlc3Npb24gb3BlbmVkIGZvciB1c2VyIHJvb3QgYnkgKHVpZD0wKQ=="
},
{
"column": "ZmFtMTpob3N0",
"timestamp": 1369725741471,
"$": "MGE5MDgxOGE="
},
{
"column": "ZmFtMTp0aW1lc3RhbXA=",
"timestamp": 1369725741470,
"$": "MTM2OTcyNTczNjY2MQ=="
}
]
},
{
"key": "cmxvZy0xMzY5NzI1NzM2NjkyLTBhOTA4MThh",
"Cell": [
{
"column": "ZmFtMTpGYWNpbGl0eQ==",
"timestamp": 1369725741508,
"$": "ODA="
},
{
"column": "ZmFtMTpGYWNpbGl0eQ==",
"timestamp": 1369725741502,
"$": "ODA="
},
{
"column": "ZmFtMTpTZXZlcml0eQ==",
"timestamp": 1369725741505,
"$": "Ng=="
},
{
"column": "ZmFtMTpTZXZlcml0eQ==",
"timestamp": 1369725741500,
"$": "Ng=="
},
{
"column": "ZmFtMTphZ2VudA==",
"timestamp": 1369725741506,
"$": "MTAuMTQ0LjEyOS4xMzc="
},
{
"column": "ZmFtMTphZ2VudA==",
"timestamp": 1369725741502,
"$": "MTAuMTQ0LjEyOS4xMzc="
},
{
"column": "ZmFtMTpib2R5",
"timestamp": 1369725741509,
"$": "c3NoZFsxMjg2OF06IHBhbV91bml4KHNzaGQ6c2Vzc2lvbik6IHNlc3Npb24gY2xvc2VkIGZvciB1c2VyIHJvb3Q="
},
{
"column": "ZmFtMTpib2R5",
"timestamp": 1369725741503,
"$": "c3NoZFsxMjg2OF06IFJlY2VpdmVkIGRpc2Nvbm5lY3QgZnJvbSAxMC4xNDQuMTI5LjEzNzogMTE6IGRpc2Nvbm5lY3RlZCBieSB1c2Vy"
},
{
"column": "ZmFtMTpob3N0",
"timestamp": 1369725741506,
"$": "MGE5MDgxOGE="
},
{
"column": "ZmFtMTpob3N0",
"timestamp": 1369725741501,
"$": "MGE5MDgxOGE="
},
{
"column": "ZmFtMTp0aW1lc3RhbXA=",
"timestamp": 1369725741504,
"$": "MTM2OTcyNTczNjY5Mg=="
},
{
"column": "ZmFtMTp0aW1lc3RhbXA=",
"timestamp": 1369725741499,
"$": "MTM2OTcyNTczNjY5MA=="
}
]
}
]
}

$ curl -s -H "Accept: application/json" 0a908189.cht.local:51111/t1/rlog-13697257366*/fam1:host

{
"Row": [
{
"key": "cmxvZy0xMzY5NzI1NzM2NjU5LTBhOTA4MThh",
"Cell": [
{
"column": "ZmFtMTpob3N0",
"timestamp": 1369725741469,
"$": "MGE5MDgxOGE="
}
]
},
{
"key": "cmxvZy0xMzY5NzI1NzM2NjYxLTBhOTA4MThh",
"Cell": [
{
"column": "ZmFtMTpob3N0",
"timestamp": 1369725741471,
"$": "MGE5MDgxOGE="
}
]
},
{
"key": "cmxvZy0xMzY5NzI1NzM2NjkyLTBhOTA4MThh",
"Cell": [
{
"column": "ZmFtMTpob3N0",
"timestamp": 1369725741506,
"$": "MGE5MDgxOGE="
},
{
"column": "ZmFtMTpob3N0",
"timestamp": 1369725741501,
"$": "MGE5MDgxOGE="
}
]
}
]
}

修改flume agent程式將rowkey格式改為${rowPrefix}${log-src-host}-${suffix}後, 查詢結果
$ curl -s -H "Accept: application/json" http://0a908189.cht.local:51111/t1/rlog-0a90818c*/

{
"Row": [
{
"key": "cmxvZy0wYTkwODE4Yy0xMzY5Nzk5NzgwMjg3",
"Cell": [
{
"column": "ZmFtMTpGYWNpbGl0eQ==",
"timestamp": 1369799786201,
"$": "ODA="
},
{
"column": "ZmFtMTpTZXZlcml0eQ==",
"timestamp": 1369799786207,
"$": "Ng=="
},
{
"column": "ZmFtMTphZ2VudA==",
"timestamp": 1369799786205,
"$": "MTAuMTQ0LjEyOS4xMzc="
},
{
"column": "ZmFtMTpib2R5",
"timestamp": 1369799786204,
"$": "c3NoZFsyOTc0N106IEFjY2VwdGVkIHB1YmxpY2tleSBmb3Igcm9vdCBmcm9tIDEwLjE0NC4xMjkuMTM3IHBvcnQgNTczNDYgc3NoMg=="
},
{
"column": "ZmFtMTpob3N0",
"timestamp": 1369799786199,
"$": "MGE5MDgxOGM="
},
{
"column": "ZmFtMTp0aW1lc3RhbXA=",
"timestamp": 1369799786196,
"$": "MTM2OTc5OTc4MDI4Ng=="
}
]
},
{
"key": "cmxvZy0wYTkwODE4Yy0xMzY5Nzk5NzgwMjkx",
"Cell": [
{
"column": "ZmFtMTpGYWNpbGl0eQ==",
"timestamp": 1369799786197,
"$": "ODA="
},
{
"column": "ZmFtMTpTZXZlcml0eQ==",
"timestamp": 1369799786206,
"$": "Ng=="
},
{
"column": "ZmFtMTphZ2VudA==",
"timestamp": 1369799786204,
"$": "MTAuMTQ0LjEyOS4xMzc="
},
{
"column": "ZmFtMTpib2R5",
"timestamp": 1369799786208,
"$": "c3NoZFsyOTc0N106IHBhbV91bml4KHNzaGQ6c2Vzc2lvbik6IHNlc3Npb24gb3BlbmVkIGZvciB1c2VyIHJvb3QgYnkgKHVpZD0wKQ=="
},
{
"column": "ZmFtMTpob3N0",
"timestamp": 1369799786202,
"$": "MGE5MDgxOGM="
},
{
"column": "ZmFtMTp0aW1lc3RhbXA=",
"timestamp": 1369799786200,
"$": "MTM2OTc5OTc4MDI5MA=="
}
]
},
{
"key": "cmxvZy0wYTkwODE4Yy0xMzY5ODA2OTg4Njc3",
"Cell": [
{
"column": "ZmFtMTpGYWNpbGl0eQ==",
"timestamp": 1369806994258,
"$": "ODA="
},
{
"column": "ZmFtMTpTZXZlcml0eQ==",
"timestamp": 1369806994254,
"$": "Ng=="
},
{
"column": "ZmFtMTphZ2VudA==",
"timestamp": 1369806994260,
"$": "MTAuMTQ0LjEyOS4xMzc="
},
{
"column": "ZmFtMTpib2R5",
"timestamp": 1369806994261,
"$": "c3NoZFsyOTc0N106IFJlY2VpdmVkIGRpc2Nvbm5lY3QgZnJvbSAxMC4xNDQuMTI5LjEzNzogMTE6IGRpc2Nvbm5lY3RlZCBieSB1c2Vy"
},
{
"column": "ZmFtMTpob3N0",
"timestamp": 1369806994256,
"$": "MGE5MDgxOGM="
},
{
"column": "ZmFtMTp0aW1lc3RhbXA=",
"timestamp": 1369806994237,
"$": "MTM2OTgwNjk4Mzc0Mw=="
}
]
},
{
"key": "cmxvZy0wYTkwODE4Yy0xMzY5ODA2OTg4Njc4",
"Cell": [
{
"column": "ZmFtMTpGYWNpbGl0eQ==",
"timestamp": 1369806994261,
"$": "ODA="
},
{
"column": "ZmFtMTpTZXZlcml0eQ==",
"timestamp": 1369806994255,
"$": "Ng=="
},
{
"column": "ZmFtMTphZ2VudA==",
"timestamp": 1369806994262,
"$": "MTAuMTQ0LjEyOS4xMzc="
},
{
"column": "ZmFtMTpib2R5",
"timestamp": 1369806994257,
"$": "c3NoZFsyOTc0N106IHBhbV91bml4KHNzaGQ6c2Vzc2lvbik6IHNlc3Npb24gY2xvc2VkIGZvciB1c2VyIHJvb3Q="
},
{
"column": "ZmFtMTpob3N0",
"timestamp": 1369806994259,
"$": "MGE5MDgxOGM="
},
{
"column": "ZmFtMTp0aW1lc3RhbXA=",
"timestamp": 1369806994239,
"$": "MTM2OTgwNjk4Mzc0Ng=="
}
]
}
]
}

訂閱：文章 (Atom)

2013年8月8日 星期四

設定 CDH4U3 Security for HFD/MR1

CentOS 6

network

passwordless SSH

hosts file

NTP

setup ntp client on all hosts

Kerberos

setup Kerberos client on all hosts

Step 3: If you are Using AES-256 Encryption, install the JCE Policy File on all hosts

HDFS

Step 2: Verify User Accounts and Groups in CDH4 Due to Security

Step 3: If you are Using AES-256 Encryption, install the JCE Policy File on all hosts

Step 4: Create and Deploy the Kerberos Principals and Keytab Files

Step 5: Shut Down the Cluster

Step 6.1: Enable Hadoop Security

Step 7: Configure Secure HDFS

[root@0a90a489] sh scpfile.sh /etc/hadoop/conf/hdfs-site.xml

Step 10: Set Variables for Secure DataNodes

Step 12.1: Start up a DataNode

Step 12.2: Test

Step 13: Set the Sticky Bit on HDFS Directories

MapReduce 0.20

Step 1: Configure Secure MRv1

Step 2: Start up the JobTracker

Step 3.1: Start up a TaskTracker

2013年7月17日 星期三

設定 CDH3U5 Audit Log

2013年7月5日 星期五

Big Data on Cloud

2013年5月28日 星期二

Setup Stargate for Apache HBase

Reference

Deployment

Query

Query software version

Query Storage Cluster Version

Query Table List

Query Table Schema

Cell Query (Single Value)

2013年8月8日星期四

2013年7月17日星期三

2013年7月5日星期五

2013年5月28日星期二