= hadoop on master and slave --- cat /etc/hosts 192.168.20.151 c0-0 192.168.20.143 c0-1 ssh-keygen -t rsa -P "" cat .ssh/id_rsa.pub >> .ssh/authorized_keys --- [root@c0-0]# tar zxvf hadoop-0.18.1.tar.gz [root@c0-0]# mv hadoop-0.18.1 /opt/hadoop [root@c0-0]# cd /opt/hadoop/conf [root@c0-0]# vi hadoop-env.sh export JAVA_HOME=/usr/java/jdk1.5.0_16 [root@c0-0]# cp hadoop-default.xml hadoop-site.xml [root@c0-0]# vi hadoop-site.xml fs.default.name hdfs://c0-0:54310 mapred.job.tracker c0-0:54311 dfs.replication 2 [root@c0-0]# vi masters c0-0 [root@c0-0]# vi slaves c0-1 [root@c0-0]# scp -r /opt/hadoop/ c0-1:/opt/ [root@c0-0]# /opt/hadoop/bin/hadoop namenode -format [root@c0-0]# jps 29905 SecondaryNameNode 29959 JobTracker 30019 Jps 29793 NameNode [root@c0-1]# jps 13822 DataNode 13930 Jps 13886 TaskTracker [root@c0-0]# mkdir /opt/hadoop/inputs [root@c0-0]# cd /opt/hadoop [root@c0-0]# echo "jun jun jun sakai sakai" > inputs/file1 [root@c0-0]# /opt/hadoop/bin/hadoop dfs -copyFromLocal inputs inputs [root@c0-0]# /opt/hadoop/bin/hadoop jar hadoop-0.18.1-examples.jar wordcount inputs outputs [root@c0-0]# /opt/hadoop/bin/hadoop dfs -cat outputs/part-00000 jun 3 sakai 2 ==hbase [root@c0-0]# tar zxvf hbase-0.18.0.tar.gz [root@c0-0]# mv hbase-0.18.0 /opt/hbase [root@c0-0]# cd /opt/hbase/conf [root@c0-0]# vi hbase-env.sh export JAVA_HOME=/usr/java/jdk1.5.0_16 [root@c0-0]# cp hbase-default.xml hbase-site.xml [root@c0-0]# vi hbase-site.xml hbase.master c0-0:60000 hbase.rootdir hdfs://c0-0:54310/hbase [root@c0-0]# vi regionservers c0-1 [root@c0-0]# cp /opt/hadoop/conf/hadoop-site.xml /opt/hbase/conf/ [root@c0-0]# scp -r /opt/hbase c0-1:/opt/ [root@c0-0]# /opt/hbase/bin/start-hbase.sh on master and slave --- # vi /etc/profile export JAVA_HOME=/usr/java/jdk1.5.0_16 export HBASE_HOME=/opt/hbase export PATH=$PATH:$JAVA_HOME/bin export CLASSPATH=.:$JAVA_HOME/jre/lib:$JAVA_HOME/lib:$JAVA_HOME/lib/tools.jar:$HBASE_HOME/hbase-0.18.0.jar:$HBASE_HOME/lib/commons-logging-1.0.4.jar:$HBASE_HOME/lib/log4j-1.2.13.jar:$HBASE_HOME/lib/hadoop-0.18.0-core.jar source /etc/profile --- [root@c0-0]# cp /opt/hadoop/conf/hadoop-site.xml /opt/hbase/conf/ [root@c0-0]#/opt/hbase/bin/hbase shell create 'myTable', {NAME=>'myColumnFamily'} list [root@c0-0]# vi /opt/hbase/conf/MyClient.java [root@c0-0]# cd /opt/hbase/conf [root@c0-0]# cat MyClient.java import java.io.IOException; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Scanner; import org.apache.hadoop.hbase.io.BatchUpdate; import org.apache.hadoop.hbase.io.Cell; import org.apache.hadoop.hbase.io.RowResult; import org.apache.hadoop.hbase.HBaseConfiguration; public class MyClient { public static void main(String args[]) throws IOException { HBaseConfiguration config = new HBaseConfiguration(); HTable table = new HTable(config, "myTable"); BatchUpdate batchUpdate = new BatchUpdate("myRow"); batchUpdate.put("myColumnFamily:columnQualifier1", "columnQualifier1 value!".getBytes()); batchUpdate.delete("myColumnFamily:cellIWantDeleted"); table.commit(batchUpdate); Cell cell = table.get("myRow", "myColumnFamily:columnQualifier1"); String valueStr = new String(cell.getValue()); Scanner scanner = table.getScanner(new String[]{"myColumnFamily:columnQualifier1"}); RowResult rowResult = scanner.next(); while(rowResult != null) { System.out.println("Found row: " + new String(rowResult.getRow()) + " with value: " + rowResult.get("myColumnFamily:columnQualifier1".getBytes())); rowResult = scanner.next(); } for (RowResult result : scanner) { System.out.println("Found row: " + new String(result.getRow()) + " with value: " + result.get("myColumnFamily:columnQualifier1".getBytes())); } scanner.close(); } } [root@c0-0]# javac /opt/hbase/conf/MyClient.java [root@c0-0]# java MyClient [root@c0-0]# /opt/hbase/bin/hbase shell hbase(main):044:0> get 'myTable','myRow' COLUMN CELL myColumnFamily:columnQualif timestamp=1224132841597, value=columnQualifier1 value! ier1 1 row(s) in 0.0570 seconds hbase(main):044:0> scan 'myTable' hbase(main):044:0> get 'myTable', 'myRow', {TIMESTAMP => 1224135131874} = map reduce [root@c0-0]# makedir /opt/hadoop/sample [root@c0-0]# cd /opt/hadoop/sample [root@c0-0]# vi CharCount.java package sample; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.FileInputFormat; import org.apache.hadoop.mapred.FileOutputFormat; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; public class CharCount extends Configured implements Tool { public int run(String[] args) throws Exception { JobConf conf = new JobConf( getConf(), sample.CharCount.class); conf.setJobName("charcount"); conf.setOutputKeyClass(Text.class); conf.setOutputValueClass(IntWritable.class); conf.setMapperClass(sample.MapClass.class); conf.setCombinerClass(sample.Reduce.class); conf.setReducerClass(sample.Reduce.class); FileInputFormat.setInputPaths(conf, args[0]); FileOutputFormat.setOutputPath(conf, new Path(args[1])); JobClient.runJob(conf); return 0; } public static void main(String[] args) throws Exception { int res = ToolRunner.run( new Configuration(), new CharCount(), args); System.exit(res); } } [root@c0-0]# vi MapClass.java package sample; import java.io.IOException; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.MapReduceBase; import org.apache.hadoop.mapred.Mapper; import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.Reporter; public class MapClass extends MapReduceBase implements Mapper { private final static IntWritable one = new IntWritable(1); private Text c = new Text(); public void map(LongWritable key, Text value, OutputCollector output, Reporter reporter) throws IOException { String line = value.toString(); int length = line.length(); for (int i = 0; i < length; i++) { c.set(line.substring(i, i + 1)); output.collect(c, one); } } } [root@c0-0]# vi Reduce.java package sample; import java.io.IOException; import java.util.Iterator; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.MapReduceBase; import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.Reducer; import org.apache.hadoop.mapred.Reporter; public class Reduce extends MapReduceBase implements Reducer { public void reduce(Text key, Iterator values, OutputCollector output, Reporter reporter) throws IOException { int sum = 0; while (values.hasNext()) { sum += values.next().get(); } output.collect(key, new IntWritable(sum)); } } [root@c0-0]# cd /opt/hadoop [root@c0-0]# javac -cp hadoop-0.18.1-core.jar sample/*.java [root@c0-0]# jar cvf charcount.jar sample/*.class [root@c0-0]# mkdir input [root@c0-0]# cat input/a.txt Jun Sakai [root@c0-0]# cat input/b.txt Yuko Sakai Mika Sakai [root@c0-0]# ./bin/hadoop dfs -copyFromLocal input input [root@c0-0]# ./bin/hadoop sample.CharCount input output [root@c0-0]# HADOOP_CLASSPATH=charcount.jar bin/hadoop sample.CharCount input output [root@c0-0]# ./bin/hadoop dfs -cat output/part-00000