Reference
Concurrent Python Example
Reference example of using concurrent.futures
in Python >3.2. Standard
parallel programming warnings still apply (race conditions, sharing state,
inter-process communication):
import concurrent.futures
import time
def calculate(limit: int) -> int:
""" An example function that takes a long time to run """
x = 1
for i in range(1, limit):
x = (x + i) % 10
return x
def iterate(data: list[int]) -> list[int]:
""" A base case without concurrency """
return [calculate(i) for i in data]
def concurrent_submit(data: list[int]) -> list[int]:
""" Concurrency using executor.submit """
result: list[int] = []
with concurrent.futures.ProcessPoolExecutor() as executor:
futures: list[concurrent.futures.Future[int]] = []
for i in data:
future = executor.submit(calculate, i)
futures.append(future)
for future in futures:
result.append(future.result())
return result
def concurrent_map(data: list[int]) -> list[int]:
""" Concurrency using executor.map """
with concurrent.futures.ProcessPoolExecutor() as executor:
# Use list to check for exceptions
data = list(executor.map(calculate, data))
return data
def main() -> None:
""" Execute each of the implementations and time it """
data = [10**7 + x for x in range(20)]
print(data)
"""
# Non-parallel implementation
start = time.time()
output = iterate(data)
duration = time.time() - start
print('iterate: %f' % duration)
print(output)
"""
# Parallel with executor.submit()
start = time.time()
output = concurrent_submit(data)
duration = time.time() - start
print('concurrent_submit: %f' % duration)
print(output)
# Parallel with executor.map()
start = time.time()
output = concurrent_map(data)
duration = time.time() - start
print('concurrent_map: %f' % duration)
print(output)
if __name__ == '__main__':
main()