Skip to content

Instantly share code, notes, and snippets.

@ixna
Last active November 29, 2016 07:29
Show Gist options
  • Save ixna/0d77a2bbc31f621522ec6fac075aa561 to your computer and use it in GitHub Desktop.
Save ixna/0d77a2bbc31f621522ec6fac075aa561 to your computer and use it in GitHub Desktop.
Get rss while doing other things concurrently
import time
import requests
from multiprocessing.pool import ThreadPool
from threading import Thread, current_thread
# Compatibility python2 (Queue.Queue) dengan python3 (queue.Queue)
try:
from queue import Queue
except:
from Queue import Queue
RSS_LIST = [
'http://feeds.nytimes.com/nyt/rss/HomePage',
'http://www.washingtonpost.com/rss/',
'http://hosted.ap.org/lineups/USHEADS-rss_2.0.xml?SITE=RANDOM&SECTION=HOME',
'http://www.npr.org/rss/rss.php?id=1001',
'http://feeds.reuters.com/reuters/topNews']
def some_process():
time.sleep(1)
def worker(rss_address):
print("Processing address %s from %s \n" % (rss_address, current_thread().getName()))
response = requests.get(rss_address, timeout=3)
# Some process that takes 1 second
some_process()
return response.content
def run_parallel():
# Batasi jumlah pool of worker 5
worker_pool = ThreadPool(5)
# Menggunakan map,
result = worker_pool.map(worker, RSS_LIST)
return result
def process_user_input(q):
while True:
user_input = q.get()
print("User input from %s: \n %s" % (current_thread().getName(), user_input))
# Some process that takes 1 second
some_process()
q.task_done()
if __name__ == "__main__":
print("Jalankan concurrent process")
# Menggunakan Queue sebagai jalur komunikasi antar thread
# (dalam aplikasi ini antara thread utama dengan thread daemon)
q = Queue()
# get rss secara concurrent, dilempar ke concurrent thread
Thread(target=run_parallel).start()
# Processing input dari user dilempar sebagai daemon thread,
# juga berjalan secara concurrent
process_input = Thread(target=process_user_input, args=(q,))
process_input.daemon = True
process_input.start()
# Main thread get input user
while True:
try:
user_input = str(input("Ketik text: "))
except:
user_input = raw_input("Ketik text:")
q.put(user_input)
# Memastikan semua thread menyelesaikan task-nya terlebih dahulu
q.join()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment