Skip to content

Instantly share code, notes, and snippets.

@developmentalmadness
Created July 17, 2013 21:57
Show Gist options
  • Save developmentalmadness/6024923 to your computer and use it in GitHub Desktop.
Save developmentalmadness/6024923 to your computer and use it in GitHub Desktop.
Sample to create HDInsight mapper - currently results in AggregateException (HttpRequestException)
using Microsoft.Hadoop.MapReduce;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace RestaurantUserMapper
{
class Program
{
static void Main(string[] args)
{
try
{
var cluster = new Uri("https://{cluster-name}.azurehdinsight.net:563");
var hadoop = Hadoop.Connect(clusterName: cluster,
userName: "admin",
password: "{password}");
Console.WriteLine("Starting: {0}", DateTime.Now);
var result = hadoop.MapReduceJob.ExecuteJob<UserRestaurantExtractionJob>();
var info = result.Info;
Console.WriteLine("Done: {0}", DateTime.Now);
Console.WriteLine("\nInfo From Server\n----------------------");
Console.WriteLine("StandardError: " + info.StandardError);
Console.WriteLine("\n----------------------");
Console.WriteLine("StandardOut: " + info.StandardOut);
Console.WriteLine("\n----------------------");
Console.WriteLine("ExitCode: " + info.ExitCode);
}
catch (Exception ex)
{
Console.WriteLine("Error: {0}", ex.Message);
Console.WriteLine("StackTrace: {0}", ex.StackTrace);
}
Console.WriteLine("Press any key to exit....");
Console.ReadKey();
}
}
public class UserRestaurantMapper : MapperBase
{
public override void Map(string inputLine, MapperContext context)
{
var data = inputLine.Split(',');
int userId, restaurantId;
if (int.TryParse(data[0], out userId) && int.TryParse(data[1], out restaurantId))
{
context.EmitLine(userId + "," + restaurantId);
}
else
{
Console.WriteLine("Unable to parse input line: '{0}'", inputLine);
}
}
}
public class UserRestaurantExtractionJob : HadoopJob<UserRestaurantMapper>
{
public override HadoopJobConfiguration Configure(ExecutorContext context)
{
var config = new HadoopJobConfiguration()
{
DeleteOutputFolder = true,
InputPath = "/recommendations/restaurant",
OutputFolder = "/recommendations/restaurant/mappingoutput"
};
return config;
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment