Mercurial > repos > bgruening > run_jupyter_job

annotate main.py @ 2:6708db9ee47e draft default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit c21261bb8373090c26cf5195134b30538b5bc714
author bgruening
date Fri, 20 Jan 2023 10:48:52 +0000
parents ff9bb9df06a7
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
1 import argparse
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
2 import os
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
3 import subprocess
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
4 import warnings
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
5 from zipfile import ZipFile
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
6
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
7 import h5py
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
8 import yaml
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
9 from skl2onnx import convert_sklearn
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
10 from skl2onnx.common.data_types import FloatTensorType
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
11
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
12
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
13 warnings.filterwarnings("ignore")
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
14 os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
15
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
16 SKLEARN_MODELS = [
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
17 "sklearn.ensemble",
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
18 "sklearn.tree",
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
19 "sklearn.linear_model",
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
20 "sklearn.svm",
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
21 "sklearn.neighbors",
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
22 "sklearn.preprocessing",
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
23 "sklearn.cluster"
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
24 ]
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
25
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
26 TF_MODELS = [
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
27 "tensorflow.python.keras.engine.training.Model",
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
28 "tensorflow.python.keras.engine.sequential.Sequential",
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
29 "tensorflow.python.keras.engine.functional.Functional",
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
30 "tensorflow.python.keras.layers",
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
31 "keras.engine.functional.Functional",
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
32 "keras.engine.sequential.Sequential",
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
33 "keras.engine.training.Model",
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
34 "keras.layers"
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
35 ]
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
36
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
37 ARRAYS = [
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
38 "numpy.ndarray",
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
39 "list"
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
40 ]
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
41
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
42 DATAFRAME = [
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
43 "pandas.core.frame.DataFrame"
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
44 ]
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
45
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
46 SCALAR_TYPES = [
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
47 "int",
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
48 "float",
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
49 "str"
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
50 ]
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
51
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
52
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
53 def find_replace_paths(script_file, updated_data_dict):
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
54 for item in updated_data_dict:
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
55 g_path = updated_data_dict[item]
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
56 script_file = script_file.replace(item, g_path)
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
57 return script_file
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
58
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
59
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
60 def update_ml_files_paths(old_file_paths, new_file_paths):
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
61 if old_file_paths == "" or old_file_paths is None or new_file_paths == "" or new_file_paths is None:
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
62 return dict()
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
63 o_files = old_file_paths.split(",")
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
64 n_files = new_file_paths.split(",")
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
65 new_paths_dict = dict()
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
66 for i, o_f in enumerate(o_files):
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
67 new_paths_dict[o_f] = n_files[i]
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
68 return new_paths_dict
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
69
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
70
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
71 def read_loaded_file(new_paths_dict, p_loaded_file, a_file, w_dir, z_file):
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
72 global_vars = dict()
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
73 input_file = yaml.safe_load(p_loaded_file)
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
74 code_string = open(input_file, "r").read()
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
75 re_code_string = find_replace_paths(code_string, new_paths_dict)
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
76 compiled_code = compile(re_code_string, input_file, 'exec')
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
77 exec(compiled_code, global_vars)
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
78 check_vars(w_dir, global_vars, a_file)
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
79 zip_files(w_dir, z_file)
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
80
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
81
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
82 def zip_files(w_dir, z_file):
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
83 with ZipFile(z_file, 'w') as zip_file:
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
84 for f_path in os.listdir(w_dir):
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
85 zip_file.write(f_path)
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
86
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
87
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
88 def create_model_path(curr_path, key):
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
89 onnx_path = curr_path + "/model_outputs"
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
90 if not os.path.exists(onnx_path):
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
91 os.makedirs(onnx_path)
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
92 onnx_model_path = curr_path + "/model_outputs/" + "onnx_model_{}.onnx".format(key)
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
93 return onnx_model_path
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
94
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
95
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
96 def save_sklearn_model(w_dir, key, obj):
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
97 initial_type = [('float_input', FloatTensorType([None, 4]))]
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
98 onx = convert_sklearn(obj, initial_types=initial_type)
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
99 sk_model_path = create_model_path(w_dir, key)
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
100 with open(sk_model_path, "wb") as f:
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
101 f.write(onx.SerializeToString())
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
102
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
103
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
104 def save_tf_model(w_dir, key, obj):
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
105 import tensorflow as tf
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
106 tf_file_key = "tf_model_{}".format(key)
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
107 tf_model_path = "{}/{}".format(w_dir, tf_file_key)
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
108 if not os.path.exists(tf_model_path):
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
109 os.makedirs(tf_model_path)
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
110 # save model as tf model
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
111 tf.saved_model.save(obj, tf_model_path)
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
112 # save model as ONNX
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
113 tf_onnx_model_p = create_model_path(w_dir, key)
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
114 # OPSET level defines a level of tensorflow operations supported by ONNX
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
115 python_shell_script = "python -m tf2onnx.convert --saved-model " + tf_model_path + " --output " + tf_onnx_model_p + " --opset 15 "
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
116 # convert tf/keras model to ONNX and save it to output file
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
117 subprocess.run(python_shell_script, shell=True, check=True)
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
118
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
119
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
120 def save_primitives(payload, a_file):
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
121 hf_file = h5py.File(a_file, "w")
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
122 for key in payload:
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
123 try:
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
124 hf_file.create_dataset(key, data=payload[key])
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
125 except Exception as e:
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
126 print(e)
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
127 continue
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
128 hf_file.close()
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
129
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
130
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
131 def save_dataframe(payload, a_file):
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
132 for key in payload:
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
133 payload[key].to_hdf(a_file, key=key)
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
134
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
135
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
136 def check_vars(w_dir, var_dict, a_file):
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
137 if var_dict is not None:
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
138 primitive_payload = dict()
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
139 dataframe_payload = dict()
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
140 for key in var_dict:
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
141 obj = var_dict[key]
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
142 obj_class = str(obj.__class__)
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
143 # save tf model
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
144 if len([item for item in TF_MODELS if item in obj_class]) > 0:
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
145 save_tf_model(w_dir, key, obj)
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
146 # save scikit-learn model
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
147 elif len([item for item in SKLEARN_MODELS if item in obj_class]) > 0:
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
148 save_sklearn_model(w_dir, key, obj)
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
149 # save arrays and lists
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
150 elif len([item for item in ARRAYS if item in obj_class]) > 0:
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
151 if key not in primitive_payload:
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
152 primitive_payload[key] = obj
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
153 elif len([item for item in DATAFRAME if item in obj_class]) > 0:
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
154 if key not in dataframe_payload:
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
155 dataframe_payload[key] = obj
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
156 elif len([item for item in SCALAR_TYPES if item in obj_class]) > 0:
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
157 if key not in primitive_payload:
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
158 primitive_payload[key] = obj
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
159 save_primitives(primitive_payload, a_file)
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
160 save_dataframe(dataframe_payload, a_file)
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
161
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
162
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
163 if __name__ == "__main__":
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
164
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
165 arg_parser = argparse.ArgumentParser()
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
166 arg_parser.add_argument("-mlp", "--ml_paths", required=True, help="")
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
167 arg_parser.add_argument("-ldf", "--loaded_file", required=True, help="")
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
168 arg_parser.add_argument("-wd", "--working_dir", required=True, help="")
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
169 arg_parser.add_argument("-oz", "--output_zip", required=True, help="")
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
170 arg_parser.add_argument("-oa", "--output_array", required=True, help="")
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
171 arg_parser.add_argument("-mlf", "--ml_h5_files", required=True, help="")
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
172 # get argument values
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
173 args = vars(arg_parser.parse_args())
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
174 ml_paths = args["ml_paths"]
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
175 loaded_file = args["loaded_file"]
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
176 array_output_file = args["output_array"]
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
177 zip_output_file = args["output_zip"]
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
178 working_dir = args["working_dir"]
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
179 ml_h5_files = args["ml_h5_files"]
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
180 new_paths_dict = update_ml_files_paths(ml_paths, ml_h5_files)
ff9bb9df06a7 "planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/jupyter_job commit f945b1bff5008ba01da31c7de64e5326579394d6"
bgruening
parents:
diff changeset
181 read_loaded_file(new_paths_dict, loaded_file, array_output_file, working_dir, zip_output_file)