
    e!hӪ                     d   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
mZ d dlZd dlmZ d dlmZmZmZmZmZmZ d dlmZ d dlmZ d dlmZmZ d dlmZ d dl m!Z!m"Z"m#Z#m$Z$m%Z% d d	l&m'Z'm(Z(m)Z) d
 Z*d Z+d Z,d Z-d Z.e%e$d               Z/e%e$ e)ej                  dk7  d      d                      Z0e%e$d               Z1e%d        Z2e%e$ e'deejf                  gddg      d                      Z4e%e$ e'deejf                  gddg      d                      Z5e%e$ e'dddg      d                      Z6e%e$ e'dddg      d                      Z7e%e$ e'dddg      d                      Z8e%e$ e'dddg      d                       Z9e%e$d!               Z:e%e$d"               Z;e%e$ e'dddg      d#                      Z<e%e$ e'dddg      d$                      Z=e%e$ e'dddg      d%                      Z>e%e$ e'deejf                  gddg      d&                      Z?e%e$ e'd ej                  dej                  j                  d'      (      dg      d)                      ZCe%e$ e'deejf                  gddg      d*                      ZDe%e$e# e'deejf                  gddg      d+                             ZEe%e$e# e'deejf                  gddg      d,                             ZFe%e$ e'deejf                  gddg      d-                      ZGd. ZHe%e$ e'deejf                  gddg      d/                      ZId0 ZJe%e$ e'd1 eK ej                  eejf                  g eMd2                   ej                  ddg eNeO eMd2                  D  cg c]  \  } }d3j                  | |       c}}       d4                      ZQd5 ZRd6 ZSe% e)ej                  dk(  d7       e'd8d9d:g      d;                      ZTe%d<        ZUd= ZVe%e$d>               ZWe%e$d?               ZXyc c}} w )@    N)sleep)ArrayMemmapForwardReducer_get_backing_memmap_get_temp_dir_strided_from_memmap_WeakArrayKeyMaphas_shareable_memory)make_memmap)_TestingMemmappingExecutor)Paralleldelayed)MemmappingPool)IS_GIL_DISABLEDnpwith_dev_shmwith_multiprocessing
with_numpy)parametrizeraisesskipifc                  2    t        j                  dd       y )Ni,  T)timeoutexit)faulthandlerdump_traceback_later     c/var/www/html/diagnosisapp-backend/venv/lib/python3.12/site-packages/joblib/test/test_memmapping.pysetup_moduler   &   s    %%c=r   c                  ,    t        j                          y N)r   cancel_dump_traceback_laterr   r   r   teardown_moduler#   *   s    ,,.r   c                      t        |       J | S r!   )r   )arrays    r   check_memmap_and_send_backr&   .   s    u%111Lr   c                 V    | \  }}}t         j                  j                  ||   |       y)zDummy helper function to be executed in subprocesses

    Check that the provided array has the expected values in the provided
    range.

    Nr   testingassert_array_equalargsdatapositionexpecteds       r   check_arrayr0   3   s(      $D(HJJ!!$x.(;r   c                     | \  }}}||   |k(  sJ ||xx   dz  cc<   t         j                  j                  ||   d|z         y)zDummy helper function to be executed in subprocesses


    Check that the input array has the right values in the provided range
    and perform an inplace modification to double the values in the range by
    two.

       Nr(   r+   s       r   inplace_doubler3   >   sL      $D(H>X%%%NaNJJ!!$x.!h,?r   c                    t         j                  j                  }| j                  d      j                  }t        j
                  |t         j                  dd      }dt        j                  |j                  d   |j                        z  |dd |j                          t        j
                  |t         j                  d	d
dd      }t        j                  d      j                  |j                        |dd |ddddddf   }|j                  dd      }t        j                  |      }|j                  }t        d| j                  dd      fd}	 |	|      }
t        |
      sJ t!        |
t         j
                        sJ  ||
|        |	|      }t        |      sJ  |||        |	|      }t        |      sJ  |||        |	|      }t!        |t         j
                        rJ t        |      sJ  |||        |	|      }t!        |t         j
                        rJ t        |      sJ  |||       |dz  }t        |      rJ  |	|      }t        |      rJ t!        |t         j
                        rJ  |||dz         t        j                  |      }t        |      rJ  |	|      }t!        |t         j"                        sJ t        |      rJ  |||       y)z9Check that it is possible to reduce a memmap backed array	test.mmapi  w+dtypeshapemodeg      r   r8   N)         r+Fr>   )r8   r9   r:   orderoffset<      r2   
   2   cTc                 $     |       \  }} || S r!   r   xconsr,   reducers      r   reconstruct_array_or_memmapzEtest_memmap_based_array_reducing.<locals>.reconstruct_array_or_memmaps       QZ
dT{r   r<   )r   r)   r*   joinstrpathmemmapfloat64aranger9   r8   flushreshapeasarrayTr   r	   
isinstancendarray)tmpdirr*   filenamebufferabb2rH   drN   a_reconstructedb_reconstructedb2_reconstructedc_reconstructedd_reconstructeda3a3_reconstructedb3b3_reconstructedrM   s                      @r    test_memmap_based_array_reducingrk   M   s    66{{;'//H YYxrzz4HF ryyaEEF1I
LLN 			

)$cRS	A 99R=  )AaD
 	
!B$"ac/A 
B	B 	

1A	A (fnnc4HG
 2!4O000oryy111* 2!4O000* 326 0111', 2!4O/299555000*1!4O/299555000* 
QB#B'''226#$4555*BII666'Q/ 
BB#B'''226&

333#$4555',r   win32z2PermissionError only easily triggerable on Windows)reasonc                    | j                  d      j                  }dj                  |      }t        j                  t
        j                  d|gt        j                  t        j                        }|j                          |j                         \  }}|j                  dk(  sJ |j                                |dk(  sJ dj                  |      }||j                         v sJ y )	Nr5   a  if 1:
    import os
    import numpy as np
    import time
    from joblib.externals.loky.backend import resource_tracker
    resource_tracker.VERBOSE = 1

    # Start the resource tracker
    resource_tracker.ensure_running()
    time.sleep(1)

    # Create a file containing numpy data
    memmap = np.memmap(r"{filename}", dtype=np.float64, shape=10, mode='w+')
    memmap[:] = np.arange(10).astype(np.int8).data
    memmap.flush()
    assert os.path.exists(r"{filename}")
    del memmap

    # Create a np.memmap backed by this file
    memmap = np.memmap(r"{filename}", dtype=np.float64, shape=10, mode='w+')
    resource_tracker.register(r"{filename}", "file")

    # Ask the resource_tracker to delete the file backing the np.memmap , this
    # should raise PermissionError that the resource_tracker will log.
    resource_tracker.maybe_unlink(r"{filename}", "file")

    # Wait for the resource_tracker to process the maybe_unlink before cleaning
    # up the memmap
    time.sleep(2)
    )r\   -cstderrstdoutr   r   z'tried to unlink {}, got PermissionError)rP   rQ   format
subprocessPopensys
executablePIPEwaitcommunicate
returncodedecode)r[   r\   cmdpouterrmsgs          r   2test_resource_tracker_retries_when_permissionerrorr      s     {{;'//H: 	!; < 		s#JOOJOO	A FFH}}HC<<1*cjjl*#::
3
:
:8
DC#**,r   c                    t         j                  j                  }| j                  d      j                  }t        j
                  |t         j                  dd      }t        j                  d      j                  |j                        |d d  |dd }|d d ddf   }|d d d d d d df   }|d	d
d   }t        d | j                  dd      fd} ||      }	t        |	      sJ t        |	t         j
                        sJ  ||	|        ||      }
t        |
      sJ  ||
|        ||      }t        |      sJ  |||        ||      }t        |      sJ  |||        ||      }t        |      sJ  |||       y )Nr5   )d      r   r<   r6   r7   i r   rF   r=   rD   r<   r>   rH   Tc                 $     |       \  }} || S r!   r   rJ   s      r   rN   zNtest_high_dimension_memmap_array_reducing.<locals>.reconstruct_array_or_memmap   rO   r   )r   r)   r*   rP   rQ   rR   rS   rT   rV   r9   r   r	   rY   )r[   r*   r\   r^   r_   rH   ra   erN   rb   rc   re   rf   e_reconstructedrM   s                 @r   )test_high_dimension_memmap_array_reducingr      s    66{{;'//H 			("**4D4PA99&'//8AaD 	
!BA	!QrT'
A	!Q1*A	!Aa%A (fnnc4HG 2!4O000oryy111*1!4O000*1!4O000*1!4O000*1!4O000*r   c                    | j                  d      j                  }dt        j                  z  }t        j                  dz   }t	        j
                  |d||z         }t        |dd|d|d d d	
	      }t        |t        j
                        sJ |j                  |k(  sJ t        |dd|d|dz  fd|d	
	      }t        |      j                  |k(  sJ y )Nr5   r=   rD   r6   )r:   r9   uint8rCF)r8   r:   rB   rA   r9   stridestotal_buffer_lenunlink_on_gc_collectr2   )r2   )
rP   rQ   mmapALLOCATIONGRANULARITYr   rR   r   rY   rB   r   )r[   fnamesizerB   
memmap_objmemmap_backed_objs         r   test__strided_from_memmapr     s    KK$,,Et)))D''!+F5t4&=AJ%"
J j")),,,&&&,qyl"
 0188FBBBr   factorymultiprocessingloky)idsc                 v   t         j                  j                  }|j                  d      j                  } | dd|      }	 |j                  d      j                  }t        j                  |t         j                  dd      }|j                  d	       |j                  t        t        |j                  d
         D cg c]&  }t        |j                  d         D ]	  }|||fd	f ( c}}        ||dt        j                  |j                        z         t        j                  |t         j                  dd      }	|j                  t        t        |	j                  d
         D cg c]&  }t        |	j                  d         D ]	  }|	||fdf ( c}}       t        j                  |      g k(  sJ  ||dt        j                  |j                        z          ||	dt        j                  |	j                        z         t        j                  |t         j                  ddd      }
t!        t"              5  |j                  t$        t        |
j                  d
         D cg c]  }|
|df	 c}       ddd       t!        t&        t(        f      5  |j                  t        t        |
j                  d
         D cg c]  }|
|df	 c}       ddd       |j+                          ~yc c}}w c c}}w c c}w # 1 sw Y   xY wc c}w # 1 sw Y   =xY w# |j+                          ~w xY w)z@Check that subprocess can access and update shared memory memmappoolrF   r2   
max_nbytestemp_folderr5   r<   r=   r6   r7         ?r   rD   )r=   r<   rH   g       @rF   r      )r8   r9   r:   rB   g      @N)r   r)   r*   mkdirrQ   rP   rR   float32fillmapr3   ranger9   onesoslistdirr   AssertionErrorr0   RuntimeError
ValueError	terminate)r   r[   r*   pool_temp_folderr~   r\   r^   ijr_   rH   s              r   test_pool_with_memmapr   (  s    66 ||F+33q.>?A);;{+33IIhbjjTJ	s	',QWWQZ'8T!%PQ
BSTQa!QTT	

 	1a"''!''"223 IIhbjjSI	',QWWQZ'8T!%PQ
BSTQa!QTT	
 zz*+r111 	1a"''!''"2231a"''!''"223 IIhbjjCPUVN# 	IEE+U1771:5FGAsGH	I
 \:./ 	LEE.aggaj8I"J1Aq#;"JK	L 	
E U U  H	I 	I #K	L 	L 	
s   A>L%  +K7+BL% .+K=B+L% +L/L;LL% +L
LLL% 7L% LLL% LL"L% %L8c                    t         j                  j                  }|j                  d      j                  } | dd|      }	 |j                  d      j                  }t        j                  |t         j                  dd      }|j                  d	       t        j                  |      }t        |t         j                        rJ t        |      sJ |j                  t        t        |j                  d
         D 	cg c]&  }t        |j                  d         D ]	  }	|||	fd	f ( c}	}        ||dt        j                   |j                        z          ||dt        j                   |j                        z         t#        j$                  |      g k(  sJ 	 |j'                          ~yc c}	}w # |j'                          ~w xY w)z?Check that subprocess can access and update shared memory arrayr   rF   r2   r   r5   r   r6   r7   r   r   rD   N)r   r)   r*   r   rQ   rP   rR   r   r   rW   rY   r	   r   r3   r   r9   r   r   r   r   )
r   r[   r*   r   r~   r\   r^   a_viewr   r   s
             r    test_pool_with_memmap_array_viewr   b  s    66 ||F+33q.>?A;;{+33IIhbjjTJ	s Afbii000#F+++	 qwwqz*qwwqz*  !Q%%	
 	1a"''!''"22361rwwqww'7#78 zz*+r111 	
#  	
s    B<F6 >+F0)A4F6 0F6 6G	backendc                 h   dj                  |       }t        j                  t        j                  d|gt        j
                  t        j
                        }|j                          |j                         \  }}|j                  dk(  s)J |j                         dz   |j                         z          y )N  if 1:
        import numpy as np
        from joblib import Parallel, delayed


        data = np.random.rand(int(2e6)).reshape((int(1e6), 2))

        # Build a complex cyclic reference that is likely to delay garbage
        # collection of the memmapped array in the worker processes.
        first_list = current_list = [data]
        for i in range(10):
            current_list = [current_list]
        first_list.append(current_list)

        if __name__ == "__main__":
            results = Parallel(n_jobs=2, backend="{b}")(
                delayed(len)(current_list) for i in range(10))
            assert results == [1] * 10
    r_   ro   rp   r   

)
rs   rt   ru   rv   rw   rx   ry   rz   r{   r|   r   r}   r~   r   r   s        r   -test_permission_error_windows_reference_cycler     s    $ 	% & 		s#JOOJOO	A FFH}}HC<<1BcjjlV3cjjlBBr   c                    dj                  |       }t        d      D ]  }t        j                  j	                         }t        j
                  j                  t              |d<   t        j                  t        j                  d|gt        j                  t        j                  |      }|j                          |j                         \  }}|j                  dk(  sJ |       |dk(  sJ d	|vrJ  y )
Nak  if 1:
        import os
        import time

        import numpy as np

        from joblib import Parallel, delayed
        from testutils import return_slice_of_data

        data = np.ones(int(2e6))

        if __name__ == '__main__':
            # warm-up call to launch the workers and start the resource_tracker
            _ = Parallel(n_jobs=2, verbose=5, backend='{b}')(
                delayed(id)(i) for i in range(20))

            time.sleep(0.5)

            slice_of_data = Parallel(n_jobs=2, verbose=5, backend='{b}')(
                delayed(return_slice_of_data)(data, 0, 20) for _ in range(10))
    r   r<   
PYTHONPATHro   rq   rr   envr   r      resource_tracker)rs   r   r   environcopypathdirname__file__rt   ru   rv   rw   rx   ry   rz   r{   )r   r}   _r   r~   r   r   s          r   3test_permission_error_windows_memmap_sent_to_parentr     s    ( 	) , 1X .jjooGGOOH5L^^T3'????	
 	
==?S||q %#% czz"#---.r   c                 d   t        j                  t        d             t        d| d      fdt	        d      D              \  } t        d| d      fdt	        d      D              \  }t
        j                  j                  |      t
        j                  j                  |      k7  sJ y )N      Y@r2   rF   n_jobsr   r   c              3   J   K   | ]  } t        t              d         ywr\   Nr   getattr.0r   r%   s     r   	<genexpr>z6test_parallel_isolated_temp_folders.<locals>.<genexpr>  '      F01
+F    #rD   c              3   J   K   | ]  } t        t              d         ywr   r   r   s     r   r   z6test_parallel_isolated_temp_folders.<locals>.<genexpr>  r   r   r   rT   intr   r   r   r   r   )r   
filename_1
filename_2r%   s      @r   #test_parallel_isolated_temp_foldersr     s     IIc#hEE81g"E F5:1XF LZ F81g"E F5:1XF LZ 77??:&"''//**EEEEr   c                 z   t        j                  t        d            t        d| d      5 } |fdt	        d      D              \  } |fdt	        d      D              \  }d d d        t
        j                  j                        t
        j                  j                        k(  sJ y # 1 sw Y   KxY w)Nr   r2   rF   r   c              3   J   K   | ]  } t        t              d         ywr   r   r   s     r   r   z9test_managed_backend_reuse_temp_folder.<locals>.<genexpr>        O))%<Or   rD   c              3   J   K   | ]  } t        t              d         ywr   r   r   s     r   r   z9test_managed_backend_reuse_temp_folder.<locals>.<genexpr>  r   r   r   )r   r~   r   r   r%   s       @r   &test_managed_backend_reuse_temp_folderr     s    
 IIc#hE	G	; PqOeAhOOOeAhOOP 77??:&"''//**EEEEP Ps   9B11B:c                     t        j                  t        d            } t               }t               }d }t	        j
                  || |f      }t	        j
                  || |f      }|j                          |j                          |j                          |j                          t        |      dk(  sJ t        |      dk(  sJ ||k7  sJ y )Nr   c                     t        ddd      5 }t        d      D ]L  } | fdt        d      D              \  }|j                  t        j                  j                  |             N 	 d d d        y # 1 sw Y   y xY w)Nr   r2   rF   )r   r   r   c              3   J   K   | ]  } t        t              d         ywr   r   r   s     r   r   z]test_memmapping_temp_folder_thread_safety.<locals>.concurrent_get_filename.<locals>.<genexpr>  s      Uq/ww/zBUr   rD   )r   r   addr   r   r   )r%   	temp_dirsr~   r   r\   s   `    r   concurrent_get_filenamezJtest_memmapping_temp_folder_thread_safety.<locals>.concurrent_get_filename  si    fQ2> 	9!2Y 9UERSHUU
bggooh789	9 	9 	9s   AA55A>)targetr,   rD   )	r   rT   r   set	threadingThreadstartrP   len)r%   temp_dirs_thread_1temp_dirs_thread_2r   t1t2s         r   )test_memmapping_temp_folder_thread_safetyr      s     IIc#hE9 
		&e5G-H
B 
		&e5G-H
B HHJHHJGGIGGI!"a'''!"a'''!3333r   c            	         d} g d}|D ]  \  }}}t        j                  t        j                  d| j	                  ||      gt         j
                  t         j
                        }|j                          |j                         \  }}|j                  |k(  sJ |j                                d|vrJ |j                                 y )Na>  if 1:
        import os
        import numpy as np
        from joblib import Parallel, delayed
        from joblib.externals.loky.backend import resource_tracker
        from concurrent.futures import ThreadPoolExecutor, wait

        resource_tracker.VERBOSE = 0

        array = np.arange(int(1e2))

        temp_dirs_thread_1 = set()
        temp_dirs_thread_2 = set()


        def raise_error(array):
            raise ValueError


        def parallel_get_filename(array, temp_dirs):
            with Parallel(backend="loky", n_jobs=2, max_nbytes=10) as p:
                for i in range(10):
                    [filename] = p(
                        delayed(getattr)(array, "filename") for _ in range(1)
                    )
                    temp_dirs.add(os.path.dirname(filename))


        def parallel_raise(array, temp_dirs):
            with Parallel(backend="loky", n_jobs=2, max_nbytes=10) as p:
                for i in range(10):
                    [filename] = p(
                        delayed(raise_error)(array) for _ in range(1)
                    )
                    temp_dirs.add(os.path.dirname(filename))


        executor = ThreadPoolExecutor(max_workers=2)

        # both function calls will use the same loky executor, but with a
        # different Parallel object.
        future_1 = executor.submit({f1}, array, temp_dirs_thread_1)
        future_2 = executor.submit({f2}, array, temp_dirs_thread_2)

        # Wait for both threads to terminate their backend
        wait([future_1, future_2])

        future_1.result()
        future_2.result()
    ))parallel_get_filenamer   r   )r   parallel_raiserD   )r   r   rD   ro   )f1f2rp   r   )
rt   ru   rv   rw   rs   rx   ry   rz   r{   r|   )r}   functions_and_returncodesr   r   r{   r~   r   r   s           r   ?test_multithreaded_parallel_termination_resource_tracker_silentr   &  s    1Cd! 8 	<B
^^T3:::#;<????

 	
3||z)73::<7)"#-;szz|;-	<r   c                 
   dj                  |       }t        j                  j                         }t        j                  j                  t              |d<   t        j                  t        j                  d|gt        j                  t        j                  |      }|j                          |j                         \  }}|j                  dk(  sJ |j                                |dk(  sJ |j                                d|vsJ y )	Na  if 1:
        import os
        import time

        import numpy as np

        from joblib import Parallel, delayed
        from testutils import return_slice_of_data

        data = np.ones(100)

        if __name__ == '__main__':
            for i in range(5):
                slice_of_data = Parallel(
                    n_jobs=2, max_nbytes=1, backend='{b}')(
                        delayed(return_slice_of_data)(data, 0, 20)
                        for _ in range(10)
                    )
    r   r   ro   r   r   r   r   )rs   r   r   r   r   r   r   rt   ru   rv   rw   rx   ry   rz   r{   r|   )r   r}   r   r~   r   r   s         r   'test_many_parallel_calls_on_same_objectr   q  s    $ 	% & **//
C1C	s#		A FFH}}HC<<1*cjjl*#:#szz|#:c)))r   c                     t        j                  t        d             t        d| d      fdt	        d      D              \  }t        |      J y )Ng     @@r2   r   r   c              3   H   K   | ]  } t        t                      y wr!   )r   r&   )r   r   r-   s     r   r   z8test_memmap_returned_as_regular_array.<locals>.<genexpr>  s&      C67+*+D1C   "rD   )r   r   r   r   r   r   )r   resultr-   s     @r   %test_memmap_returned_as_regular_arrayr    sT     773s8DBxq'cB C;@8C HV v&...r   c                    | dk(  r4t         j                  j                  d      rt        j                  d       dj                  |       }t        j                  t         j                  d|gt        j                  t        j                        }|j                          |j                         \  }}|j                         }|j                         }|j                  dk(  sJ |d	z   |z          d
|vsJ |       y )Nr   winzVThe temporary folder cannot be deleted on Windows in the presence of a reference cycler   r   ro   rp   r   r   resource_tracker)rv   platform
startswithpytestxfailrs   rt   ru   rw   rx   ry   rz   r|   r{   r   s        r   2test_resource_tracker_silent_when_reference_cyclesr	    s    " &S\\44U; 	,	

$ 	% & 		s#JOOJOO	A FFH}}HC
**,C
**,C<<10cFlS00S(-#-(r   c           	      Z   t        j                  |j                        g k(  sJ  | dd|j                  d      }	 t        j                  |j                        g k(  sJ t         j                  j	                  |j
                        rJ t        j                  dt        j                        }|j                  dk(  sJ |j                  t        t        |j                  d         D cg c]  }||d	f	 c}       t        j                  |j                        g k(  sJ t        j                  d
t        j                        }|j                  dk(  sJ |j                  t        t        |j                  d         D cg c]  }||d	f	 c}       t         j                  j                  |j
                        sJ t        j                  |j
                        }t!        |      dk(  sJ t        j"                  dgd
z  d      }|j                  t$        |g      }|d   rJ 	 |j'                          t        d      D ]9  }t)        d       t         j                  j	                  |j
                        r8 ~y t+        dj-                  |j
                              c c}w c c}w # |j'                          t        d      D ]9  }t)        d       t         j                  j	                  |j
                        r8 ~w  t+        dj-                  |j
                              xY w)z0Check that large arrays are not copied in memoryr<   (   r2   )r   r   verboser=   r;   r   r   r   r      rD   abcobjectrF   皙?z#temporary folder {} was not deletedN)r   r   rQ   r   exists_temp_folderr   r   r   nbytesr   r0   r   r9   rS   isdirr   r%   r	   r   r   r   rs   )	r   r[   r~   smallr   largedumped_filenamesobjectsresultss	            r   %test_memmapping_pool_for_large_arraysr    s    ::fnn%+++ 	bfnnaHA(zz&..)R///77>>!..111,||r!!!	kU5;;q>5JKUAsOKL zz&..)R/// 2::.||s"""	kU5;;q>5JKUAsOKL ww}}Q^^,,,::ann5#$))) ((E7S=9%%,wi81:~: 	
r 	A#J77>>!..1
 	
 !5<<Q^^L = L L  	
r 	A#J77>>!..1
 	
 !5<<Q^^L s3   B/J. +J$
7BJ. J)
BJ. $
J. .AL*(L*z,https://github.com/joblib/joblib/issues/1086)marksc                 n   dj                  |       }t        j                  j                         }t        j                  j                  t              |d<   t        j                  t        j                  d|gt        j                  t        j                  |      }|j                          |j                         \  }}|j                         |j                         }}|j                  d      d   }|j                   dk(  sJ |xs |       |dk(  sJ t        j                  j#                  |      rJ y )	Na  if 1:
        import os
        from pathlib import Path
        from time import sleep

        import numpy as np
        from joblib import Parallel, delayed
        from testutils import print_filename_and_raise

        data = np.random.rand(1000)

        def get_temp_folder(parallel_obj, backend):
            if "{b}" == "loky":
                return Path(parallel_obj._backend._workers._temp_folder)
            else:
                return Path(parallel_obj._backend._pool._temp_folder)


        if __name__ == "__main__":
            try:
                with Parallel(n_jobs=2, backend="{b}", max_nbytes=100) as p:
                    temp_folder = get_temp_folder(p, "{b}")
                    p(delayed(print_filename_and_raise)(data)
                              for i in range(1))
            except ValueError as e:
                # the temporary folder should be deleted by the end of this
                # call but apparently on some file systems, this takes
                # some time to be visible.
                #
                # We attempt to write into the temporary folder to test for
                # its existence and we wait for a maximum of 10 seconds.
                for i in range(100):
                    try:
                        with open(temp_folder / "some_file.txt", "w") as f:
                            f.write("some content")
                    except FileNotFoundError:
                        # temp_folder has been deleted, all is fine
                        break

                    # ... else, wait a bit and try again
                    sleep(.1)
                else:
                    raise AssertionError(
                        str(temp_folder) + " was not deleted"
                    ) from e
    r   r   ro   r   
r    )rs   r   r   r   r   r   r   rt   ru   rv   rw   rx   ry   rz   r|   splitr{   r  )r   r}   r   r~   r   r   r\   s          r   &test_child_raises_parent_exits_cleanlyr     s    :-Z 	[ \ **//
C1C	s#		A FFH}}HCzz|SZZ\Cyyq!H<<1(cjS("99ww~~h''''r   c           	           | dd|j                         }	 t        j                  |j                         g k(  sJ t        j                  dt        j
                        }|j                  dk(  sJ |j                  t        t        |j                  d         D cg c]  }||df	 c}       t        j                  |j                         g k(  sJ 	 |j                          ~yc c}w # |j                          ~w xY w)	z2Check that large arrays memmapping can be disabledr<   Nr   r   r;   r  r   r   )rQ   r   r   r   r   rS   r  r   r0   r   r9   r   )r   r[   r~   r  r   s        r   .test_memmapping_pool_for_large_arrays_disabledr"  w  s     	d?Azz&..)R/// 2::.||s"""	kU5;;q>5JKUAsOKL zz&..)R/// 	
 L 	
s   BC* C%
'+C* %C* *C=c                    t         j                  }	 t        d      t         _         | dd      }	 |j                  }d}|j	                  |      sJ t
        j                  j                  |      sJ t        j                  dt        j                        }|j                  dk(  sJ |j                  t        |gdz         t        t        j                  |            d	k(  sJ t        j                  dt        j                        d
z  }|j                  dk(  sJ |j                  t        |gdz         t        t        j                  |            d
k(  sJ 	 |j!                          ~t#        d      D ].  }t
        j                  j                  |      s nt%        d       0 t'        d      	 |t         _        y# |j!                          ~w xY w# |t         _        w xY w)z1Check that memmapping uses /dev/shm when possibleg    ~Ar<   rF   r   z"/dev/shm/joblib_memmapping_folder_r   r;   r  rD   r2   r  z(temporary folder of pool was not deletedN)jmrSYSTEM_SHARED_MEM_FS_MIN_SIZEr   r  r  r   r   r  r   r   rS   r  r   idr   r   r   r   r   r   )r   	orig_sizer~   r   folder_prefixr^   r_   r   s           r   'test_memmapping_on_large_enough_dev_shmr*    s    11I+6 -0I)A"%	  !~~@M#..}===77>>"2333 2::.A88s?"?EE"qcBh rzz"234999
 2::.2A88s?"?EE"qcBhrzz"234999 KKMs 	MA77>>"23#J		M !!KLL 
 -6) KKM -6)s$   G D+G AG GG G%c                 l   t         j                  }	 t        d      t         _         | dd      }	 |j                  }|j	                  d      rJ 	 |j                          ~t        j                  j                  |      rJ 	 |t         _        y # |j                          ~w xY w# |t         _        w xY w)Ng@6Dr<   rF   r$  z/dev/shm)	r%  r&  r   r  r  r   r   r   r  )r   r(  r~   r   s       r   $test_memmapping_on_too_small_dev_shmr,    s     11I6 -0J)A"%	  !~~'22:>>>> KKM 77>>"23333,5) KKM
 -6)s"   B& B 2B& B##B& &B3c                 f   t         j                  j                  } | dd|j                        }	 |j	                  t         j
                  d      }|j                         }t        |      rJ  ||t        j
                  d             |j                          ~y# |j                          ~w xY w)z:Check that large arrays are not copied in memory in returnr<   rF   r   )  r,   r.  N)	r   r)   r*   rQ   apply_asyncr   getr	   r   )r   r[   r*   r~   resr  s         r   /test_memmapping_pool_for_large_arrays_in_returnr3    s     66 	bfnn=AmmBGG'm2	'...5"''$-0	 	
s   AB B0c                 &    t        |       sJ | |z  S )z4Multiplication function to be executed by subprocess)r	   )r^   n_timess     r   _worker_multiplyr6  
  s    """w;r   c                    t         j                  j                  } | dd|j                        }	 t        j                  t        j
                  d      j                  d      d      dddd	ddf   }|j                  t        |df
      j                         }t        |      rJ  ||d|z         |j                          ~y# |j                          ~w xY w)zCheck that memmaps with a bad buffer are returned as regular arrays

    Unary operations and ufuncs on memmap instances return a new memmap
    instance with an in-memory buffer (probably a numpy bug).
    r<   rF   r   ip  )r.  r2   r<   r@   )rA   NrD   r/  )r   r)   r*   rQ   rW   rT   rV   r0  r6  r1  r	   r   )r   r[   r*   r~   r^   r_   s         r   6test_workaround_against_bad_memmap_with_copied_buffersr8    s     66bfnn=A JJryy..|<CHBQBPQR MM*!QM8<<>'***1a!e$	 	
s   BC Cc                     | S r!   r   )args    r   identityr;  /  s    Jr   zfactory,retry_nor<   z{}, {}c                    |j                  d      j                  }dt        j                  z  }t        j                  dz   }t	        |d|d|      } | d|j                        }|j                  t        |f	      j                         }t        |t        j                        sJ |j                  |k(  sJ t        j                  j                  ||       |j                          y )
Nr5   r=   rD   r6   r   )r:   r9   r8   rB   r2   r   r/  )rP   rQ   r   r   r
   r0  r;  r1  rY   r   rR   rB   r)   r*   r   )	r   retry_nor[   r   r   rB   objr~   r   s	            r    test_pool_memmap_with_big_offsetr@  3  s    & KK$,,Et)))D''!+F
e$d'&
QCv~~.A]]83&]1557Ffbii(((==F"""JJ!!#v.KKMr   c                    d}t        || j                        \  }}|du sJ || j                  d      j                  k(  sJ t        |d       \  }}t        j                  j                  d      r|du sJ |j                  |      sJ y )Ntest.tmpdirFr=  r  )r   rQ   rP   rv   r  r  endswith)r[   pool_folder_namepool_folder
shared_mems       r   test_pool_get_temp_dirrG  S  s    $+,<fnnMK&++m4<<<<<+,<$OK
||u%U""" 0111r   c                 ,   d}ddl }t        |j                  j                  d      r&|j	                  |j                  j                  d       t        |d      \  }}t        j                  j                  d      r|du sJ |j                  |      sJ y)z`Check that _get_temp_dir works when os.statvfs is not defined

    Regression test for #902
    rB  r   Nstatvfsr=  r  F)
joblib._memmapping_reducerhasattr_memmapping_reducerr   delattrr   rv   r  r  rC  )r[   monkeypatchrD  joblibrE  rF  s         r   !test_pool_get_temp_dir_no_statvfsrP  _  s    
 %%v)),,i8F66999E+,<$OK
||u%U""" 0111r   z1This test fails with a PermissionError on Windows	mmap_moder?   r6   c                 "   d t        d      D cg c]  }t        j                  dd       }} t        | dd      fd	t	        |      D              }t	        |      D ]%  \  }}t        j
                  j                  ||       ' y c c}w )
Nc                     || d d  | S r!   r   )arrvalues     r   funcz4test_numpy_arrays_use_different_memory.<locals>.funcw  s    A
r   rF   )rF   rF   rS   r;   r   r2   )rQ  r   r   c              3   H   K   | ]  \  }} t              ||        y wr!   r   )r   r   rT  rV  s      r   r   z9test_numpy_arrays_use_different_memory.<locals>.<genexpr>}  s)      D"(!Sc1Dr   )r   r   zerosr   	enumerater)   r*   )rQ  r   arraysr  rT  rV  s        @r   &test_numpy_arrays_use_different_memoryr\  q  s     <A9Eabhhxy1EFEChqC D,5f,=D G G$ .3


%%c1-. Fs   Bc            	         dd} t        j                  d      }t               }|j                  |d       |j	                  |      dk(  sJ |}|j	                  |      dk(  sJ |j                  |d       |j	                  |      dk(  sJ ~t        j                          t        |j                        dk(  sJ |j	                  |      dk(  sJ ~ | |j                         t        j                  d      }|j                  |d       t        |j                        dk(  sJ |j	                  |      dk(  sJ t        t              5  |j	                  t        j                  d             d d d        ~ | |j                         d }t        t        d	      D cg c]  } |||       c}      }t        j                         d
k(  rt        rdnd}t        |      |k  sJ y y # 1 sw Y   {xY wc c}w )Nr   c                     t        |      D ]1  }t        |       dk(  r y t        j                          t	        d       3 t        |       dk(  sJ y )Nr   r  )r   r   gccollectr   )	containerretriesr   s      r   assert_empty_after_gc_collectz>test_weak_array_key_map.<locals>.assert_empty_after_gc_collect  sH    w 	A9~"JJL#J		
 9~"""r   *   r^   r_   rD   rH   c                     t        j                  d      }t        t              5  | j	                  |       d d d        | j                  ||       | j	                  |      |k(  sJ t        |      S # 1 sw Y   <xY w)Nrd  )r   r   r   KeyErrorr1  r   r'  )mr   r^   s      r   get_set_get_collectz4test_weak_array_key_map.<locals>.get_set_get_collect  sa    GGBKH 	EE!H		auuQx1}}!u		 	s   A22A;r.  CPythoni  )r   )r   r   r   r   r1  r_  r`  r   _datar   rf  r   r  python_implementationr   )	rc  r^   rg  r_   rH   rh  r   
unique_idsmax_len_unique_idss	            r   test_weak_array_key_maprn    s   # 	AAEE!SM558s??	A558s??EE!SM558s??	JJLqww<1558s??	!!''*
AEE!SMqww<1558s??		 	bggbk 	
!!''* tEA)!Q/EFJ%%'94 %4S:!3333 5#   Fs   
%G)G5)G2c                      t               } t        t        j                        5  t        j                  |        d d d        y # 1 sw Y   y xY wr!   )r   r   picklePicklingErrordumps)rg  s    r   #test_weak_array_key_map_no_picklingrs    s8    A	$$	% Q  s   AAc                    t        | j                  d            t        j                  dd      }|j	                         fdd         } t        d      fd	|fD              } t        d      fd
|fD              }t        j                  j                  ||        t        d      fdt        d      D              }t        j                  j                  |d   |       y )Nzarr.datrF   r   r;   c                      t              5 } t        j                  | j                         dt        j                  d      }d d d        t	        j
                  dt        j                  d      S # 1 sw Y   0xY w)Nr   )accessrB   r   )r8   r]   rB   )openr   filenoACCESS_READr   rZ   r   )fdmmtestfiles     r   _read_arrayz%test_direct_mmap.<locals>._read_array  s^    (^ 	Nr299;$2B2B1MB	Nzz%rxx1EE	N 	Ns   6A22A;c                     | dz  S Nr2   r   rK   s    r   rV  ztest_direct_mmap.<locals>.func      !tr   r2   r   c              3   @   K   | ]  } t              |        y wr!   rX  r   rK   rV  s     r   r   z#test_direct_mmap.<locals>.<genexpr>       ;!]WT]1-;   c              3   @   K   | ]  } t              |        y wr!   rX  r  s     r   r   z#test_direct_mmap.<locals>.<genexpr>        Aaq!1 Ar  c              3   >   K   | ]  } t                       y wr!   rX  r   r   r}  s     r   r   z#test_direct_mmap.<locals>.<genexpr>        JA!5!5!7 J   rD   r   )	strrP   r   rT   tofiler   r)   r*   r   r[   r^   rT  refr  r}  rV  r|  s        @@@r   test_direct_mmapr    s     6;;y)*H
		"G$AHHXF
 -C (!
;s;
;C !ha  AC5 AAGJJ!!'3/ !ha  Jq JJGJJ!!'!*c2r   c                 2   t        | j                  d            t        j                  dd      j	                  dd      }|j                         fdd         } t        d	      fd
|fD              } t        d	      fd|fD              }|d   j                  d   rJ t        j                  j                  ||        t        d	      fdt        d      D              }t        j                  j                  |d   |       y )Nzarr2.datrF   r   r;   r=   r2   c                  R    t        j                        } | j                  dd      S )Nr=   r2   )r   rR   rV   )r{  r|  s    r   r}  z=test_parallel_memmap2d_as_memmap_1d_base.<locals>._read_array  s"    YYx zz!Qr   c                     | dz  S r  r   r  s    r   rV  z6test_parallel_memmap2d_as_memmap_1d_base.<locals>.func  r  r   r  c              3   @   K   | ]  } t              |        y wr!   rX  r  s     r   r   z;test_parallel_memmap2d_as_memmap_1d_base.<locals>.<genexpr>  r  r  c              3   @   K   | ]  } t              |        y wr!   rX  r  s     r   r   z;test_parallel_memmap2d_as_memmap_1d_base.<locals>.<genexpr>  r  r  r   F_CONTIGUOUSc              3   >   K   | ]  } t                       y wr!   rX  r  s     r   r   z;test_parallel_memmap2d_as_memmap_1d_base.<locals>.<genexpr>  r  r  rD   )r  rP   r   rT   rV   r  r   flagsr)   r*   r   r  s        @@@r   (test_parallel_memmap2d_as_memmap_1d_baser    s    
 6;;z*+H
		"G$,,Q2AHHX  -C (!
;s;
;C !ha  AC5 AAGqz///JJ!!'3/ !ha  Jq JJGJJ!!'!*a0r   )Yr   r_  	itertoolsr   r   rp  r  rt   rv   r   timer   r  rJ  rL  r%  r   r   r   r   r   r	   joblib.backportsr
   joblib.executorr   TestExecutorjoblib.parallelr   r   joblib.poolr   joblib.test.commonr   r   r   r   r   joblib.testingr   r   r   r   r#   r&   r0   r3   rk   r   r   r   get_memmapping_executorr   r   r   r   r   r   r   r   r   r  r	  r  parammarkr  r   r"  r*  r,  r3  r6  r8  r;  listproductr   r   r  rs   r@  rG  rP  r\  rn  rs  r  r  )rK   ys   00r   <module>r     sr    	   	    
    (  ) F - &  7 6>/
<@ S-  S-l LLG$X*  
*Z )+  )+X C CD \99:	F#
0  0f \99:	F#
$  $N Y*F34C 5  CB Y*F34*. 5  *.Z Y*F34
F 5  
F Y*F34F 5  F !4  !4H F<  F<R Y*F34%* 5  %*P Y*F34/ 5  / Y*F342. 5  2.j \99:	F#
1  1h ++##E $ 	
 	K(  K(\ \99:	F#
  , \99:	F#
.6   .6b \99:	F#
6   60 \99:	F#
  , \99:	F#
  0 	\AABE!H	
 &I%%'8&&A3sERSHCUV	Aq 	1	  $	22$ LLG$W [4,'. ( 
. 84 84v 3  36 1  1U	s   P,