Created
February 21, 2012 19:31
-
-
Save sgmurphy/1878352 to your computer and use it in GitHub Desktop.
Scrape ratings from iTunes store
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<?php | |
/** | |
* Scrape the number of podcast reviews from iTunes for all country specific storefronts. | |
* | |
* @author Sean Murphy <[email protected]> | |
*/ | |
$podcast_id = '366931951'; // Startups For the Rest of Us | |
//$podcast_id = '318567721'; // techzing | |
$countries = json_decode('[{"storefront":"143455-6,12","name":"Canada"},{"storefront":"143441-1,12","name":"United States"},{"storefront":"143565,12","name":"Belarus"},{"storefront":"143446-2,12","name":"Belgium"},{"storefront":"143526,12","name":"Bulgaria"},{"storefront":"143494,12","name":"Croatia"},{"storefront":"143557-2,12","name":"Cyprus"},{"storefront":"143489,12","name":"Czech Republic"},{"storefront":"143458-2,12","name":"Denmark"},{"storefront":"143443,12","name":"Deutschland"},{"storefront":"143454-8,12","name":"España"},{"storefront":"143518,12","name":"Estonia"},{"storefront":"143447-2,12","name":"Finland"},{"storefront":"143442,12","name":"France"},{"storefront":"143448,12","name":"Greece"},{"storefront":"143482,12","name":"Hungary"},{"storefront":"143558,12","name":"Iceland"},{"storefront":"143449,12","name":"Ireland"},{"storefront":"143450,12","name":"Italia"},{"storefront":"143519,12","name":"Latvia"},{"storefront":"143520,12","name":"Lithuania"},{"storefront":"143451-2,12","name":"Luxembourg"},{"storefront":"143530,12","name":"Macedonia"},{"storefront":"143521,12","name":"Malta"},{"storefront":"143523,12","name":"Moldova"},{"storefront":"143452,12","name":"Nederland"},{"storefront":"143457-2,12","name":"Norway"},{"storefront":"143445,12","name":"Österreich"},{"storefront":"143478,12","name":"Poland"},{"storefront":"143453,12","name":"Portugal"},{"storefront":"143487,12","name":"Romania"},{"storefront":"143496,12","name":"Slovakia"},{"storefront":"143499,12","name":"Slovenia"},{"storefront":"143456,12","name":"Sverige"},{"storefront":"143459-2,12","name":"Switzerland"},{"storefront":"143480,12","name":"Turkey"},{"storefront":"143444,12","name":"United Kingdom"},{"storefront":"143469,12","name":"Россия"},{"storefront":"143563,12","name":"Algeria"},{"storefront":"143564,12","name":"Angola"},{"storefront":"143524,12","name":"Armenia"},{"storefront":"143568,12","name":"Azerbaijan"},{"storefront":"143559,12","name":"Bahrain"},{"storefront":"143525,12","name":"Botswana"},{"storefront":"143516,12","name":"Egypt"},{"storefront":"143573,12","name":"Ghana"},{"storefront":"143467,12","name":"India"},{"storefront":"143491,12","name":"Israel"},{"storefront":"143528,12","name":"Jordan"},{"storefront":"143529,12","name":"Kenya"},{"storefront":"143493,12","name":"Kuwait"},{"storefront":"143497,12","name":"Lebanon"},{"storefront":"143531,12","name":"Madagascar"},{"storefront":"143532,12","name":"Mali"},{"storefront":"143533,12","name":"Mauritius"},{"storefront":"143534,12","name":"Niger"},{"storefront":"143561,12","name":"Nigeria"},{"storefront":"143562,12","name":"Oman"},{"storefront":"143498,12","name":"Qatar"},{"storefront":"143479,12","name":"Saudi Arabia"},{"storefront":"143535,12","name":"Senegal"},{"storefront":"143472,12","name":"South Africa"},{"storefront":"143572,12","name":"Tanzania"},{"storefront":"143536,12","name":"Tunisia"},{"storefront":"143481,12","name":"UAE"},{"storefront":"143537,12","name":"Uganda"},{"storefront":"143571,12","name":"Yemen"},{"storefront":"143460,12","name":"Australia"},{"storefront":"143560,12","name":"Brunei Darussalam"},{"storefront":"143465-2,12","name":"China"},{"storefront":"143463,12","name":"Hong Kong"},{"storefront":"143476,12","name":"Indonesia"},{"storefront":"143462-1,12","name":"Japan"},{"storefront":"143517,12","name":"Kazakhstan"},{"storefront":"143515,12","name":"Macau"},{"storefront":"143473,12","name":"Malaysia"},{"storefront":"143461,12","name":"New Zealand"},{"storefront":"143477,12","name":"Pakistan"},{"storefront":"143474,12","name":"Philippines"},{"storefront":"143464,12","name":"Singapore"},{"storefront":"143486,12","name":"Sri Lanka"},{"storefront":"143470,12","name":"Taiwan"},{"storefront":"143475,12","name":"Thailand"},{"storefront":"143566,12","name":"Uzbekistan"},{"storefront":"143471,12","name":"Vietnam"},{"storefront":"143466,12","name":"대한민국"},{"storefront":"143538,12","name":"Anguilla"},{"storefront":"143540,12","name":"Antigua and Barbuda"},{"storefront":"143505-2,12","name":"Argentina"},{"storefront":"143539,12","name":"Bahamas"},{"storefront":"143541,12","name":"Barbados"},{"storefront":"143555-2,12","name":"Belize"},{"storefront":"143542,12","name":"Bermuda"},{"storefront":"143556-2,12","name":"Bolivia"},{"storefront":"143503,12","name":"Brasil"},{"storefront":"143543,12","name":"British Virgin Islands"},{"storefront":"143544,12","name":"Cayman Islands"},{"storefront":"143483-2,12","name":"Chile"},{"storefront":"143501-2,12","name":"Colombia"},{"storefront":"143495-2,12","name":"Costa Rica"},{"storefront":"143545,12","name":"Dominica"},{"storefront":"143508-2,12","name":"Dominican Republic"},{"storefront":"143509-2,12","name":"Ecuador"},{"storefront":"143506-2,12","name":"El Salvador"},{"storefront":"143546,12","name":"Grenada"},{"storefront":"143504-2,12","name":"Guatemala"},{"storefront":"143553,12","name":"Guyana"},{"storefront":"143510-2,12","name":"Honduras"},{"storefront":"143511,12","name":"Jamaica"},{"storefront":"143468,12","name":"México"},{"storefront":"143547,12","name":"Montserrat"},{"storefront":"143512-2,12","name":"Nicaragua"},{"storefront":"143485-2,12","name":"Panama"},{"storefront":"143513-2,12","name":"Paraguay"},{"storefront":"143507-2,12","name":"Peru"},{"storefront":"143548,12","name":"St. Kitts and Nevis"},{"storefront":"143549,12","name":"St. Lucia"},{"storefront":"143550,12","name":"St. Vincent & The Grenadines"},{"storefront":"143554-2,12","name":"Suriname"},{"storefront":"143551,12","name":"Trinidad and Tobago"},{"storefront":"143552,12","name":"Turks & Caicos"},{"storefront":"143514-2,12","name":"Uruguay"},{"storefront":"143502-2,12","name":"Venezuela"}]'); | |
$ratings = array(); | |
foreach ($countries as $country) { | |
$ch = curl_init(); | |
curl_setopt($ch, CURLOPT_URL, "http://itunes.apple.com/WebObjects/MZStore.woa/wa/customerReviews?displayable-kind=4&id={$podcast_id}"); | |
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); | |
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0); | |
curl_setopt($ch, CURLOPT_HTTPHEADER, array( | |
'User-Agent: iTunes/10.3.1 (Macintosh; Intel Mac OS X 10.6.8) AppleWebKit/533.21.1', | |
"X-Apple-Store-Front: {$country->storefront}", | |
'X-Apple-Tz: -18000', | |
'Accept-Language: en-us, en;q=0.50', | |
)); | |
$body = curl_exec($ch); | |
curl_close($ch); | |
$dom = new DOMDocument(); | |
@$dom->loadHTML($body); | |
$xpath = new DOMXPath($dom); | |
$nodes = $xpath->query("//span[@class='rating-count']"); | |
if ($nodes->item(0)) { | |
$ratings[$country->name] = (int)preg_replace('/([\d]+)/', '$1', $nodes->item(0)->nodeValue); | |
} else { | |
$ratings[$country->name] = 0; | |
} | |
} | |
?> | |
<html> | |
<head> | |
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> | |
</head> | |
<body> | |
<table> | |
<thead> | |
<tr> | |
<th>Country</th> | |
<th>Ratings</th> | |
</tr> | |
</thead> | |
<tbody> | |
<?php | |
foreach ($ratings as $country => $count) { | |
if ($count) { | |
echo "<tr><td>{$country}</td><td>{$count}</td></tr>\n"; | |
} | |
} | |
?> | |
</tbody> | |
<tfoot> | |
<tr> | |
<td><strong>TOTAL</strong></td> | |
<td><strong><?php echo array_sum($ratings); ?></strong></td> | |
</tr> | |
</tfoot> | |
</table> | |
</body> |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
@SM09E thanks for your list of language mapping. I just test the third value, the result is:
42 Mac1
41 MacTV1
40 MacAMPLibrary1
39 MacMusic1
38 MacPodcasts1
34 Configurator
32 iTunes122
30 K84
29 P84
28 iTunes12
26 P8
25 K8
24 P71
23 K71
21 P7
20 K7
17 iTunes11
16 P6
15 K6
14 EnterpriseStore
12 iTunes10
11 VolumeStore
The way I tested the third part of the value is :
https://apps.apple.com/{country code}/app/{app name}/id{appid}
Test range is 0-50, But I only got a partially determined value。I think this has something to do with the country. But i just has two account.
So i want to know, if you have any progress since then?