Pig Tutorial

# pig local mode 
pig -x local

# pig mapreduce mode
pig -x mapreduce
pig -x mapreduce Sample_script.pig

records= load 'hdfs://localhost:9000/input/sample_small.txt' using com.oserp.pigudf.WordCountLoadFunc() as (words:bag{word:(w:chararray)});

A = load '/data/cupid_algo/workspace/shanshan/dt=2017-04-16' using parquet.pig.ParquetLoader(); 
STORE records INTO ' hdfs://localhost:9000/pig_Output/ ' USING PigStorage (',');

sh ls;

# clear the screen
clear;

# show the data
DUMP records;
# show the schema
DESCRIBE records;
# show the logical, physical relations
EXPLAIN records;
# execute step by step
ILLUSTRATE records;

# groupby
Group_data = GROUP records BY age;
Group_data = GROUP records BY (age, sex);
Group_all = GROUP records BY ALL;
# COGROUP two more relations
cogroup_data = COGROUP student_details by age, employee_details by age;

# inner join
Relation3_name = JOIN Relation1_name BY key, Relation2_name BY key ;
# left join
Relation3_name = JOIN Relation1_name BY id LEFT OUTER, Relation2_name BY customer_id;
# right join
outer_right = JOIN customers BY id RIGHT, orders BY customer_id;
# full join
outer_full = JOIN customers BY id FULL OUTER, orders BY customer_id;
# multiple key
emp = JOIN employee BY (id,jobid), employee_contact BY (id,jobid);

Relation3_name = CROSS Relation1_name, Relation2_name;

student = UNION student1, student2;
SPLIT student_details into student_details1 if age<23, student_details2 if (22<age and age>25);
# delete the duplicate
distinct_data = DISTINCT student_details;

# filter data
filter_data = FILTER student_details BY city == 'Chennai';
# for each
foreach_data = FOREACH student_details GENERATE id,age,city;
# get limited data
Result = LIMIT Relation_name required number of tuples;
# order
order_by_data = ORDER student_details BY age DESC;

# eval func
student_gpa_avg = foreach student_group_all  Generate
# get average
student_gpa_avg = foreach student_group_all  Generate
   (student_details.firstname, student_details.gpa), AVG(student_details.gpa); 
# some functions
# bagtostring: transfer bag to string
# concat: two expressions
# MAX, MIN, COUNT, ISEMPTY, SUM, SUBTRACT
# TOBAG, TOTUPLE, TOMAP, TOP
# other string, numeric, date, time functions
## Execute pig Script