Created
July 17, 2013 21:57
-
-
Save developmentalmadness/6024923 to your computer and use it in GitHub Desktop.
Sample to create HDInsight mapper - currently results in AggregateException (HttpRequestException)
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
using Microsoft.Hadoop.MapReduce; | |
using System; | |
using System.Collections.Generic; | |
using System.Linq; | |
using System.Text; | |
using System.Threading.Tasks; | |
namespace RestaurantUserMapper | |
{ | |
class Program | |
{ | |
static void Main(string[] args) | |
{ | |
try | |
{ | |
var cluster = new Uri("https://{cluster-name}.azurehdinsight.net:563"); | |
var hadoop = Hadoop.Connect(clusterName: cluster, | |
userName: "admin", | |
password: "{password}"); | |
Console.WriteLine("Starting: {0}", DateTime.Now); | |
var result = hadoop.MapReduceJob.ExecuteJob<UserRestaurantExtractionJob>(); | |
var info = result.Info; | |
Console.WriteLine("Done: {0}", DateTime.Now); | |
Console.WriteLine("\nInfo From Server\n----------------------"); | |
Console.WriteLine("StandardError: " + info.StandardError); | |
Console.WriteLine("\n----------------------"); | |
Console.WriteLine("StandardOut: " + info.StandardOut); | |
Console.WriteLine("\n----------------------"); | |
Console.WriteLine("ExitCode: " + info.ExitCode); | |
} | |
catch (Exception ex) | |
{ | |
Console.WriteLine("Error: {0}", ex.Message); | |
Console.WriteLine("StackTrace: {0}", ex.StackTrace); | |
} | |
Console.WriteLine("Press any key to exit...."); | |
Console.ReadKey(); | |
} | |
} | |
public class UserRestaurantMapper : MapperBase | |
{ | |
public override void Map(string inputLine, MapperContext context) | |
{ | |
var data = inputLine.Split(','); | |
int userId, restaurantId; | |
if (int.TryParse(data[0], out userId) && int.TryParse(data[1], out restaurantId)) | |
{ | |
context.EmitLine(userId + "," + restaurantId); | |
} | |
else | |
{ | |
Console.WriteLine("Unable to parse input line: '{0}'", inputLine); | |
} | |
} | |
} | |
public class UserRestaurantExtractionJob : HadoopJob<UserRestaurantMapper> | |
{ | |
public override HadoopJobConfiguration Configure(ExecutorContext context) | |
{ | |
var config = new HadoopJobConfiguration() | |
{ | |
DeleteOutputFolder = true, | |
InputPath = "/recommendations/restaurant", | |
OutputFolder = "/recommendations/restaurant/mappingoutput" | |
}; | |
return config; | |
} | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment