Skip to content

pydvl.parallel.backends.joblib

JoblibParallelBackend 

JoblibParallelBackend(config: ParallelConfig | None = None)

Bases: ParallelBackend

Class used to wrap joblib to make it transparent to algorithms.

Example

from pydvl.parallel import JoblibParallelBackend
parallel_backend = JoblibParallelBackend()
Source code in src/pydvl/parallel/backends/joblib.py 
36
37
38
39
40
41
42
43
44
45
46
47
48
@deprecated(
    target=True,
    args_mapping={"config": None},
    deprecated_in="0.9.0",
    remove_in="0.10.0",
)
def __init__(self, config: ParallelConfig | None = None) -> None:
    n_jobs: int | None = None
    if config is not None:
        n_jobs = config.n_cpus_local
    self.config = {
        "n_jobs": n_jobs,
    }

executor classmethod 

executor(
    max_workers: int | None = None,
    *,
    config: ParallelConfig | None = None,
    cancel_futures: CancellationPolicy | bool = NONE,
) -> Executor

Returns a futures executor for the parallel backend.

Example

from pydvl.parallel import JoblibParallelBackend
parallel_backend = JoblibParallelBackend()
with parallel_backend.executor() as executor:
    executor.submit(...)
PARAMETER DESCRIPTION
max_workers

Maximum number of parallel workers.

TYPE: int | None DEFAULT: None

config

(DEPRECATED) Object configuring parallel computation, with cluster address, number of cpus, etc.

TYPE: ParallelConfig | None DEFAULT: None

cancel_futures

Policy to use when cancelling futures after exiting an Executor.

TYPE: CancellationPolicy | bool DEFAULT: NONE

RETURNS DESCRIPTION
Executor

Instance of [_ReusablePoolExecutor][joblib.externals.loky.reusable_executor].
Source code in src/pydvl/parallel/backends/joblib.py 
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
@classmethod
def executor(
    cls,
    max_workers: int | None = None,
    *,
    config: ParallelConfig | None = None,
    cancel_futures: CancellationPolicy | bool = CancellationPolicy.NONE,
) -> Executor:
    """Returns a futures executor for the parallel backend.

    !!! Example
        ``` python
        from pydvl.parallel import JoblibParallelBackend
        parallel_backend = JoblibParallelBackend()
        with parallel_backend.executor() as executor:
            executor.submit(...)
        ```

    Args:
        max_workers: Maximum number of parallel workers.
        config: (**DEPRECATED**) Object configuring parallel computation,
            with cluster address, number of cpus, etc.
        cancel_futures: Policy to use when cancelling futures
            after exiting an Executor.

    Returns:
        Instance of [_ReusablePoolExecutor][joblib.externals.loky.reusable_executor].
    """
    if config is not None:
        warnings.warn(
            "The `JoblibParallelBackend` uses deprecated arguments: "
            "`config`. They were deprecated since v0.9.0 "
            "and will be removed in v0.10.0.",
            FutureWarning,
        )

    if cancel_futures not in (CancellationPolicy.NONE, False):
        warnings.warn(
            "Cancellation of futures is not supported by the joblib backend",
        )
    return cast(Executor, get_reusable_executor(max_workers=max_workers))

wrap 

wrap(fun: Callable, **kwargs) -> Callable

Wraps a function as a joblib delayed.

PARAMETER DESCRIPTION
fun

the function to wrap

TYPE: Callable

RETURNS DESCRIPTION
Callable

The delayed function.
Source code in src/pydvl/parallel/backends/joblib.py 
 98
 99
100
101
102
103
104
105
106
107
def wrap(self, fun: Callable, **kwargs) -> Callable:
    """Wraps a function as a joblib delayed.

    Args:
        fun: the function to wrap

    Returns:
        The delayed function.
    """
    return delayed(fun)  # type: ignore