| import os |
| import argparse |
| import multiprocessing as mp |
|
|
|
|
| def fn(para): |
| file, url_base, data_dir = para |
| url = url_base + file |
| output_filename = file+".zip" |
| output_path = os.path.join(data_dir, output_filename) |
| unzip_path = os.path.join(data_dir, file) |
| os.system(f"curl -X GET {url} -H accept: */* --output {output_path}") |
| os.system(f"unzip {output_path} -d {unzip_path}") |
|
|
|
|
| if __name__ == "__main__": |
| parser = argparse.ArgumentParser() |
|
|
| parser.add_argument("--filename_file", type=str, default="./dataset/ATLAS_filename.txt") |
| parser.add_argument("--url_base", type=str, default="https://www.dsimb.inserm.fr/ATLAS/api/ATLAS/analysis/") |
| parser.add_argument("--output_dir", type=str, default="./dataset/ATLAS_test") |
|
|
| args = parser.parse_args() |
|
|
| num_processes = 48 |
|
|
| os.makedirs(args.output_dir, exist_ok=True) |
|
|
| with open(args.filename_file,'r+') as f: |
| file_cont = f.read() |
| file_list = file_cont.split("\n") |
| para_list = [(file, args.url_base, args.output_dir) for file in file_list] |
|
|
| with mp.Pool(num_processes) as pool: |
| _ = pool.map(fn, para_list) |
| print("finished") |
| |
| os.system(f"cp -rf {args.filename_file} {args.output_dir}") |
|
|
