first commit

Author: SuperFastPython

Diff for: .gitignore

@@ -0,0 +1,3 @@
+# git ignore
+**/.DS_Store
+.DS_Store

Diff for: README.md

@@ -0,0 +1,55 @@
+# Python ProcessPoolExecutor Jump-Start
+
+![Python ProcessPoolExecutor Jump-Start](cover.png)
+
+* <https://github.com/SuperFastPython/PythonProcessPoolExecutorJumpStart>
+
+This repository provides all source code for the book:
+
+* **Python ProcessPoolExecutor Jump-Start**: _Execute CPU-Bound Tasks in Parallel With Modern Process Pools_, Jason Brownlee, 2022.
+
+
+## Source Code
+You can access all Python .py files directly here:
+
+* [src/](src/)
+
+## Get the Book
+
+You can learn more about the book here:
+
+* Coming soon
+
+### Book Blurb
+
+> How much faster could your python code run (if it used all CPU cores)?
+>
+> The ProcessPoolExecutor class provides modern process pools for CPU-bound tasks.
+>
+> This is not some random third-party library, this is a class provided in the Python standard library (already installed on your system).
+>
+> This is the class you need to make your code run faster.
+>
+> There's just one problem. No one knows about it (or how to use it well).
+>
+> Introducing: "Python ProcessPoolExecutor Jump-Start". A new book designed to teach you modern process pools in Python, super fast!
+>
+> You will get a rapid-paced, 7-part course to get you started and make you awesome at using the ProcessPoolExecutor.
+>
+> Including:
+>
+> * How to create process pools and when to use them.
+> * How to configure process pools including the number of workers.
+> * How to execute tasks with worker processes and handle for results.
+> * How to execute tasks in the process pool asynchronously.
+> * How to query and get results from handles on asynchronous tasks called futures.
+> * How to wait on and manage diverse collections of asynchronous tasks.
+> * How to develop a parallel Fibonacci calculator 4x faster than the sequential version.
+>
+> Each of the 7 lessons was carefully designed to teach one critical aspect of the ProcessPoolExecutor, with explanations, code snippets and worked examples.
+>
+> Each lesson ends with an exercise for you to complete to confirm you understood the topic, a summary of what was learned, and links for further reading if you want to go deeper.
+>
+> Stop copy-pasting code from StackOverflow answers.
+>
+> Learn Python concurrency correctly, step-by-step.

Diff for: cover.png

@@ -0,0 +1,17 @@
+# SuperFastPython.com
+# example of running a function in a new process
+from multiprocessing import Process
+
+# custom function to be executed in a child process
+def task():
+    # report a message
+    print('This is another process', flush=True)
+
+# protect the entry point
+if __name__ == '__main__':
+    # define a task to run in a new process
+    process = Process(target=task)
+    # start the task in a new process
+    process.start()
+    # wait for the child process to terminate
+    process.join()

Diff for: src/lesson01_process.py

@@ -0,0 +1,18 @@
+# SuperFastPython.com
+# example running a function in the process pool
+from concurrent.futures import ProcessPoolExecutor
+
+# custom function to be executed in a worker process
+def task():
+    # report a message
+    print('This is another process', flush=True)
+
+# protect the entry point
+if __name__ == '__main__':
+    # create the process pool
+    with ProcessPoolExecutor() as exe:
+        # issue the task
+        future = exe.submit(task)
+        # wait for the task to finish
+        future.result()
+    # close the process pool automatically

Diff for: src/lesson01_processpoolexecutor.py

@@ -0,0 +1,12 @@
+# SuperFastPython.com
+# example reporting the details of a default pool
+from concurrent.futures import ProcessPoolExecutor
+
+# protect the entry point
+if __name__ == '__main__':
+    # create a process pool
+    exe = ProcessPoolExecutor()
+    # report the status of the process pool
+    print(exe._max_workers)
+    # shutdown the process pool
+    exe.shutdown()

Diff for: src/lesson02_default_config.py

@@ -0,0 +1,24 @@
+# SuperFastPython.com
+# example initializing worker processes in the pool
+from time import sleep
+from concurrent.futures import ProcessPoolExecutor
+
+# custom function to be executed in a worker process
+def task(number):
+    # report a message
+    print(f'Worker task {number}...', flush=True)
+    # block for a moment
+    sleep(1)
+
+# initialize a worker in the process pool
+def init():
+    # report a message
+    print('Initializing worker...', flush=True)
+
+# protect the entry point
+if __name__ == '__main__':
+    # create and configure the process pool
+    with ProcessPoolExecutor(2,
+        initializer=init) as exe:
+        # issue tasks to the process pool
+        _ = exe.map(task, range(4))

Diff for: src/lesson02_initialize_processes.py

@@ -0,0 +1,19 @@
+# SuperFastPython.com
+# example of setting a large number number of workers
+from time import sleep
+from concurrent.futures import ProcessPoolExecutor
+
+# custom task function executed in the process pool
+def task(number):
+    # block for a moment
+    sleep(1)
+    # report a message
+    if number % 10 == 0:
+        print(f'>task {number} done', flush=True)
+
+# protect the entry point
+if __name__ == '__main__':
+    # create a process pool
+    with ProcessPoolExecutor(50) as exe:
+        # issue many tasks to the pool
+        _ = exe.map(task, range(50))

Diff for: src/lesson02_num_processes.py

@@ -0,0 +1,16 @@
+# SuperFastPython.com
+# example of executing multiple tasks in chunks
+from concurrent.futures import ProcessPoolExecutor
+
+# custom function to be executed in a worker process
+def task(number):
+    return number*2
+
+# protect the entry point
+if __name__ == '__main__':
+    # create the process pool
+    with ProcessPoolExecutor(4) as exe:
+        # issue tasks to execute concurrently
+        _ = exe.map(task, range(10000), chunksize=500)
+    # wait for all tasks to complete
+    print('All done')

Diff for: src/lesson03_map_chunksize.py

@@ -0,0 +1,25 @@
+# SuperFastPython.com
+# example executing tasks concurrently with multiple arg
+from random import random
+from time import sleep
+from concurrent.futures import ProcessPoolExecutor
+
+# custom function to be executed in a worker process
+def task(number, value):
+    # report a message
+    print(f'Task using {value}', flush=True)
+    # block for a moment to simulate work
+    sleep(value)
+    # return a new value
+    return number + value
+
+# protect the entry point
+if __name__ == '__main__':
+    # create the process pool
+    with ProcessPoolExecutor(4) as exe:
+        # prepare random numbers between 0 and 1
+        values = [random() for _ in range(10)]
+        # issue tasks to execute concurrently
+        for result in exe.map(task, range(10), values):
+            # report results
+            print(result)

Diff for: src/lesson03_map_multiple_arguments.py

@@ -0,0 +1,16 @@
+# SuperFastPython.com
+# example of executing multiple tasks with no chunks
+from concurrent.futures import ProcessPoolExecutor
+
+# custom function to be executed in a worker process
+def task(number):
+    return number*2
+
+# protect the entry point
+if __name__ == '__main__':
+    # create the process pool
+    with ProcessPoolExecutor(4) as exe:
+        # issue tasks to execute concurrently
+        _ = exe.map(task, range(10000), chunksize=1)
+    # wait for all tasks to complete
+    print('All done')

Diff for: src/lesson03_map_no_chunksize.py

@@ -0,0 +1,22 @@
+# SuperFastPython.com
+# example of executing tasks concurrently with no return
+from random import random
+from time import sleep
+from concurrent.futures import ProcessPoolExecutor
+
+# custom function to be executed in a worker process
+def task(number):
+    # generate a random value between 0 and 1
+    value = random()
+    # report a message
+    print(f'Task generated {value}', flush=True)
+    # block for a moment to simulate work
+    sleep(value)
+
+# protect the entry point
+if __name__ == '__main__':
+    # create the process pool
+    with ProcessPoolExecutor(4) as exe:
+        # issue tasks to execute concurrently
+        _ = exe.map(task, range(10))
+    # wait automatically for all tasks to finish...

Diff for: src/lesson03_map_no_return.py

@@ -0,0 +1,25 @@
+# SuperFastPython.com
+# example of executing tasks concurrently with one arg
+from random import random
+from time import sleep
+from concurrent.futures import ProcessPoolExecutor
+
+# custom function to be executed in a worker process
+def task(number):
+    # generate a random value between 0 and 1
+    value = random()
+    # report a message
+    print(f'Task generated {value}', flush=True)
+    # block for a moment to simulate work
+    sleep(value)
+    # return a new value
+    return number + value
+
+# protect the entry point
+if __name__ == '__main__':
+    # create the process pool
+    with ProcessPoolExecutor(4) as exe:
+        # issue tasks to execute concurrently
+        for result in exe.map(task, range(10)):
+            # report results
+            print(result)

Diff for: src/lesson03_map_one_argument.py

@@ -0,0 +1,32 @@
+# SuperFastPython.com
+# example of executing tasks concurrently with timeout
+from random import random
+from time import sleep
+from concurrent.futures import ProcessPoolExecutor
+from concurrent.futures import TimeoutError
+
+# custom function to be executed in a worker process
+def task(number):
+    # generate a random value between 0 and 1
+    value = random()
+    # report a message
+    print(f'Task generated {value}', flush=True)
+    # block for a moment to simulate work
+    sleep(number + value)
+    # return a new value
+    return number + value
+
+# protect the entry point
+if __name__ == '__main__':
+    # create the process pool
+    with ProcessPoolExecutor(4) as exe:
+        try:
+            # issue tasks to execute concurrently
+            for result in exe.map(task, range(10),
+                timeout=2):
+                # report results
+                print(result)
+        except TimeoutError:
+            print('Gave up, took too long')
+    # report that we will wait for the tasks to complete
+    print('Waiting for tasks to complete...')

Diff for: src/lesson03_map_timeout.py

@@ -0,0 +1,26 @@
+# SuperFastPython.com
+# example issuing many asynchronous tasks systematically
+from random import random
+from time import sleep
+from concurrent.futures import ProcessPoolExecutor
+
+# custom function to be executed in a worker process
+def task(number):
+    # generate a random value between 0 and 1
+    value = random()
+    # report a message
+    print(f'Task generated {value}', flush=True)
+    # block for a moment to simulate work
+    sleep(value)
+    # return a new value
+    return number + value
+
+# protect the entry point
+if __name__ == '__main__':
+    # create the process pool
+    with ProcessPoolExecutor() as ex:
+        # issue many asynchronous tasks systematically
+        futures = [ex.submit(task, i) for i in range(5)]
+        # enumerate futures and report results
+        for future in futures:
+            print(future.result())

Diff for: src/lesson04_submit_many_tasks.py

@@ -0,0 +1,25 @@
+# SuperFastPython.com
+# example issuing an async task with multiple arguments
+from random import random
+from time import sleep
+from concurrent.futures import ProcessPoolExecutor
+
+# custom function to be executed in a worker process
+def task(number, value):
+    # report a message
+    print(f'Task received {value}', flush=True)
+    # block for a moment to simulate work
+    sleep(value)
+    # return a new value
+    return number + value
+
+# protect the entry point
+if __name__ == '__main__':
+    # create the process pool
+    with ProcessPoolExecutor() as exe:
+        # issue an asynchronous task
+        future = exe.submit(task, 100, random())
+        # get the result once the task completes
+        result = future.result()
+        # report the result
+        print(result)

Diff for: src/lesson04_submit_multiple_arg.py

@@ -0,0 +1,27 @@
+# SuperFastPython.com
+# example issuing an asynchronous task with no arguments
+from random import random
+from time import sleep
+from concurrent.futures import ProcessPoolExecutor
+
+# custom function to be executed in a worker process
+def task():
+    # generate a random value between 0 and 1
+    value = random()
+    # report a message
+    print(f'Task generated {value}', flush=True)
+    # block for a moment to simulate work
+    sleep(value)
+    # return a new value
+    return value
+
+# protect the entry point
+if __name__ == '__main__':
+    # create the process pool
+    with ProcessPoolExecutor() as exe:
+        # issue an asynchronous task
+        future = exe.submit(task)
+        # get the result once the task completes
+        result = future.result()
+        # report the result
+        print(result)