Mercurial > repos > devteam > data_manager_rsync_g2

annotate data_manager/data_manager_rsync.py @ 1:861c071a6cd5 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
author iuc
date Tue, 04 Apr 2017 17:53:15 -0400
parents 70afa70bba41
children cf2f615e2515
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
1 #!/usr/bin/env python
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
2 # Dan Blankenberg
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
3 from __future__ import print_function
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
4
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
5 import datetime
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
6 import logging
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
7 import optparse
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
8 import os
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
9 import shutil
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
10 import subprocess
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
11 import tempfile
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
12 from json import (
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
13 dumps,
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
14 loads
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
15 )
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
16 from os.path import basename
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
17 from xml.etree.ElementTree import tostring
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
18 try:
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
19 # For Python 3.0 and later
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
20 from urllib.request import urlopen
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
21 except ImportError:
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
22 # Fall back to Python 2 imports
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
23 from urllib2 import urlopen
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
24
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
25 _log_name = __name__
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
26 if _log_name == '__builtin__':
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
27 _log_name = 'toolshed.installed.g2.rsync.data.manager'
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
28 log = logging.getLogger( _log_name )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
29
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
30 # Get the Data from the Galaxy Project rsync server
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
31 RSYNC_CMD = 'rsync'
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
32 RSYNC_SERVER = "rsync://datacache.g2.bx.psu.edu/"
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
33 LOCATION_DIR = "location"
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
34 INDEX_DIR = "indexes"
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
35
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
36 # Pull the Tool Data Table files from github
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
37 # FIXME: These files should be accessible from the rsync server directly.
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
38 TOOL_DATA_TABLE_CONF_XML_URLS = {'main': "https://raw.githubusercontent.com/galaxyproject/usegalaxy-playbook/master/files/galaxy/usegalaxy.org/config/tool_data_table_conf.xml",
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
39 'test': "https://raw.githubusercontent.com/galaxyproject/usegalaxy-playbook/master/files/galaxy/test.galaxyproject.org/config/tool_data_table_conf.xml" }
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
40
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
41 # Replace data table source entries with local temporary location
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
42 GALAXY_DATA_CANONICAL_PATH = "/galaxy/data/"
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
43 TOOL_DATA_TABLE_CONF_XML_REPLACE_SOURCE = '<file path="%slocation/' % ( GALAXY_DATA_CANONICAL_PATH )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
44 TOOL_DATA_TABLE_CONF_XML_REPLACE_TARGET = '<file path="%s/'
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
45
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
46 # Some basic Caching, so we don't have to reload and download everything every time
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
47 CACHE_TIME = datetime.timedelta( minutes=10 )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
48 TOOL_DATA_TABLES_LOADED_BY_URL = {}
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
49
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
50 # Entries will not be selected by default
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
51 DEFAULT_SELECTED = False
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
52
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
53 # Exclude data managers without 'path' column or that are in the manual exclude list
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
54 PATH_COLUMN_NAMES = ['path']
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
55 EXCLUDE_DATA_TABLES = []
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
56 # TODO: Make additional handler actions available for tables that can't fit into the the basic
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
57 # "take the value of path" as a dir and copy contents.
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
58 # e.g. mafs. Although this maf table is goofy and doesn't have path defined in <table> def,
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
59 # but it does exit in the .loc.
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
60
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
61
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
62 # --- These methods are called by/within the Galaxy Application
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
63 def exec_before_job( app, inp_data, out_data, param_dict, tool=None, **kwd ):
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
64 # Look for any data tables that haven't been defined for this data manager before and dynamically add them to Galaxy
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
65 param_dict = dict( **param_dict )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
66 param_dict['data_table_entries'] = param_dict.get( 'data_table_entries', [] )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
67 if not isinstance( param_dict['data_table_entries'], list ):
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
68 param_dict['data_table_entries'] = [param_dict['data_table_entries']]
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
69 param_dict['data_table_entries'] = ",".join( param_dict['data_table_entries'] )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
70 if tool:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
71 tool_shed_repository = tool.tool_shed_repository
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
72 else:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
73 tool_shed_repository = None
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
74 tdtm = None
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
75 data_manager = app.data_managers.get_manager( tool.data_manager_id, None )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
76 data_table_entries = get_data_table_entries( param_dict )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
77 data_tables = load_data_tables_from_url( data_table_class=app.tool_data_tables.__class__ ).get( 'data_tables' )
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
78 for data_table_name, entries in data_table_entries.items():
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
79 # get data table managed by this data Manager
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
80 has_data_table = app.tool_data_tables.get_tables().get( data_table_name )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
81 if has_data_table:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
82 has_data_table = bool( has_data_table.get_filename_for_source( data_manager, None ) )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
83 if not has_data_table:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
84 if tdtm is None:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
85 from tool_shed.tools import data_table_manager
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
86 tdtm = data_table_manager.ToolDataTableManager( app )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
87 target_dir, tool_path, relative_target_dir = tdtm.get_target_install_dir( tool_shed_repository )
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
88 # Dynamically add this data table
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
89 log.debug( "Attempting to dynamically create a missing Tool Data Table named %s." % data_table_name )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
90 data_table = data_tables[data_table_name]
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
91 repo_info = tdtm.generate_repository_info_elem_from_repository( tool_shed_repository, parent_elem=None )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
92 if repo_info is not None:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
93 repo_info = tostring( repo_info )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
94 tmp_file = tempfile.NamedTemporaryFile()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
95 tmp_file.write( get_new_xml_definition( app, data_table, data_manager, repo_info, target_dir ) )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
96 tmp_file.flush()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
97 app.tool_data_tables.add_new_entries_from_config_file( tmp_file.name, None, app.config.shed_tool_data_table_config, persist=True )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
98 tmp_file.close()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
99
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
100
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
101 def galaxy_code_get_available_data_tables( trans ):
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
102 # list of data tables
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
103 found_tables = get_available_tables( trans )
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
104 rval = [ ( x, x, DEFAULT_SELECTED ) for x in found_tables]
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
105 return rval
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
106
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
107
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
108 def galaxy_code_get_available_data_tables_entries( trans, dbkey, data_table_names ):
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
109 # available entries, optionally filtered by dbkey and table names
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
110 if dbkey in [ None, '', '?' ]:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
111 dbkey = None
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
112 if data_table_names in [ None, '', '?' ]:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
113 data_table_names = None
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
114 found_tables = get_available_tables_for_dbkey( trans, dbkey, data_table_names )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
115 dbkey_text = '(%s) ' % ( dbkey ) if dbkey else ''
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
116 rval = [( "%s%s" % ( dbkey_text, x[0] ), dumps( dict( name=x[0].split( ': ' )[0], entry=x[1] ) ).encode( 'base64' ).rstrip(), DEFAULT_SELECTED ) for x in found_tables.items()]
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
117 return rval
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
118
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
119 # --- End Galaxy called Methods ---
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
120
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
121
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
122 def rsync_urljoin( base, url ):
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
123 # urlparse.urljoin doesn't work correctly for our use-case
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
124 # probably because it doesn't recognize the rsync scheme
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
125 base = base.rstrip( '/' )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
126 url = url.lstrip( '/' )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
127 return "%s/%s" % ( base, url )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
128
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
129
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
130 def rsync_list_dir( server, dir=None, skip_names=[] ):
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
131 # drwxr-xr-x 50 2014/05/16 20:58:11 .
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
132 if dir:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
133 dir = rsync_urljoin( server, dir )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
134 else:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
135 dir = server
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
136 rsync_response = tempfile.NamedTemporaryFile()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
137 rsync_stderr = tempfile.NamedTemporaryFile()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
138 rsync_cmd = [ RSYNC_CMD, '--list-only', dir ]
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
139 return_code = subprocess.call( rsync_cmd, stdout=rsync_response, stderr=rsync_stderr )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
140 rsync_response.flush()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
141 rsync_response.seek(0)
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
142 rsync_stderr.flush()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
143 rsync_stderr.seek(0)
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
144 if return_code:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
145 msg = "stdout:\n%s\nstderr:\n%s" % ( rsync_response.read(), rsync_stderr.read() )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
146 rsync_response.close()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
147 rsync_stderr.close()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
148 raise Exception( 'Failed to execute rsync command (%s), returncode=%s. Rsync_output:\n%s' % ( rsync_cmd, return_code, msg ) )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
149 rsync_stderr.close()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
150 rval = {}
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
151 for line in rsync_response:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
152 perms, line = line.split( None, 1 )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
153 line = line.strip()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
154 size, line = line.split( None, 1 )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
155 line = line.strip()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
156 date, line = line.split( None, 1 )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
157 line = line.strip()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
158 time, line = line.split( None, 1 )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
159 name = line.strip()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
160 if name in skip_names:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
161 continue
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
162 size = line.strip()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
163 rval[ name ] = dict( name=name, permissions=perms, bytes=size, date=date, time=time )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
164 rsync_response.close()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
165 return rval
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
166
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
167
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
168 def rsync_sync_to_dir( source, target ):
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
169 rsync_response = tempfile.NamedTemporaryFile()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
170 rsync_stderr = tempfile.NamedTemporaryFile()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
171 rsync_cmd = [ RSYNC_CMD, '-avzP', source, target ]
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
172 return_code = subprocess.call( rsync_cmd, stdout=rsync_response, stderr=rsync_stderr )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
173 rsync_response.flush()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
174 rsync_response.seek(0)
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
175 rsync_stderr.flush()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
176 rsync_stderr.seek(0)
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
177 if return_code:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
178 msg = "stdout:\n%s\nstderr:\n%s" % ( rsync_response.read(), rsync_stderr.read() )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
179 rsync_response.close()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
180 rsync_stderr.close()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
181 raise Exception( 'Failed to execute rsync command (%s), returncode=%s. Rsync_output:\n%s' % ( rsync_cmd, return_code, msg ) )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
182 rsync_response.close()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
183 rsync_stderr.close()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
184 return return_code
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
185
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
186
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
187 def data_table_needs_refresh( cached_data_table, url ):
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
188 if cached_data_table is None:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
189 return True, {}
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
190 if datetime.datetime.now() - cached_data_table.get( 'time_loaded' ) > CACHE_TIME:
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
191 data_table_text = urlopen( url ).read()
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
192 if cached_data_table.get( 'data_table_text', None ) != data_table_text:
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
193 return True, {'data_table_text': data_table_text}
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
194 loc_file_attrs = rsync_list_dir( RSYNC_SERVER, LOCATION_DIR )
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
195 if cached_data_table.get( 'loc_file_attrs', None ) != loc_file_attrs:
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
196 return True, {'loc_file_attrs': loc_file_attrs}
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
197 return False, {}
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
198
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
199
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
200 def load_data_tables_from_url( url=None, site='main', data_table_class=None ):
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
201 if not url:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
202 url = TOOL_DATA_TABLE_CONF_XML_URLS.get( site, None )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
203 assert url, ValueError( 'You must provide either a URL or a site=name.' )
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
204
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
205 cached_data_table = TOOL_DATA_TABLES_LOADED_BY_URL.get( url, None )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
206 refresh, attribs = data_table_needs_refresh( cached_data_table, url )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
207 if refresh:
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
208 data_table_text = attribs.get( 'data_table_text' )or urlopen( url ).read()
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
209 loc_file_attrs = attribs.get( 'loc_file_attrs' ) or rsync_list_dir( RSYNC_SERVER, LOCATION_DIR )
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
210
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
211 tmp_dir = tempfile.mkdtemp( prefix='rsync_g2_' )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
212 tmp_loc_dir = os.path.join( tmp_dir, 'location' )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
213 os.mkdir( tmp_loc_dir )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
214 rsync_sync_to_dir( rsync_urljoin( RSYNC_SERVER, LOCATION_DIR ), os.path.abspath( tmp_loc_dir ) )
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
215
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
216 new_data_table_text = data_table_text.replace( TOOL_DATA_TABLE_CONF_XML_REPLACE_SOURCE, TOOL_DATA_TABLE_CONF_XML_REPLACE_TARGET % ( tmp_loc_dir ) )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
217 data_table_fh = tempfile.NamedTemporaryFile( dir=tmp_dir, prefix='rysnc_data_manager_data_table_conf_' )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
218 data_table_fh.write( new_data_table_text )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
219 data_table_fh.flush()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
220 tmp_data_dir = os.path.join( tmp_dir, 'tool-data' )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
221 os.mkdir( tmp_data_dir )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
222 data_tables = data_table_class( tmp_data_dir, config_filename=data_table_fh.name )
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
223 for name, data_table in list(data_tables.data_tables.items()):
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
224 if name in EXCLUDE_DATA_TABLES or not data_table_has_path_column( data_table ):
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
225 log.debug( 'Removing data table "%s" because it is excluded by name or does not have a defined "path" column.', name )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
226 del data_tables.data_tables[name]
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
227 cached_data_table = { 'data_tables': data_tables, 'tmp_dir': tmp_dir, 'data_table_text': data_table_text, 'tmp_loc_dir': tmp_loc_dir, 'loc_file_attrs': loc_file_attrs, 'time_loaded': datetime.datetime.now() }
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
228 TOOL_DATA_TABLES_LOADED_BY_URL[ url ] = cached_data_table
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
229 # delete the files
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
230 data_table_fh.close()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
231 cleanup_before_exit( tmp_dir )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
232 return cached_data_table
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
233
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
234
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
235 def data_table_has_path_column( data_table ):
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
236 col_names = data_table.get_column_name_list()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
237 for name in PATH_COLUMN_NAMES:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
238 if name in col_names:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
239 return True
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
240 return False
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
241
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
242
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
243 def get_available_tables( trans ):
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
244 # list of data tables
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
245 data_tables = load_data_tables_from_url( data_table_class=trans.app.tool_data_tables.__class__ )
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
246 return list(data_tables.get( 'data_tables' ).get_tables().keys())
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
247
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
248
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
249 def get_new_xml_definition( app, data_table, data_manager, repo_info=None, location_file_dir=None ):
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
250 sub_dict = { 'table_name': data_table.name, 'comment_char': '', 'columns': '', 'file_path': '' }
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
251 sub_dict.update( data_manager.get_tool_shed_repository_info_dict() )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
252 if data_table.comment_char:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
253 sub_dict['comment_char'] = 'comment_char="%s"' % ( data_table.comment_char )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
254 for i, name in enumerate( data_table.get_column_name_list() ):
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
255 if name is not None:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
256 sub_dict['columns'] = "%s\n%s" % ( sub_dict['columns'], '<column name="%s" index="%s" />' % ( name, i ) )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
257 location_file_dir = location_file_dir or app.config.galaxy_data_manager_data_path
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
258 for filename in data_table.filenames.keys():
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
259 sub_dict['file_path'] = basename( filename )
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
260 sub_dict['file_path'] = os.path.join( location_file_dir, sub_dict['file_path'] ) # os.path.abspath?
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
261 if not os.path.exists( sub_dict['file_path'] ):
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
262 # Create empty file
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
263 open( sub_dict['file_path'], 'wb+' ).close()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
264 break
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
265 sub_dict[ 'repo_info' ] = repo_info or ''
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
266 return """
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
267 <tables><table name="%(table_name)s" %(comment_char)s>
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
268 %(columns)s
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
269 <file path="%(file_path)s" />
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
270 %(repo_info)s
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
271 </table></tables>
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
272 """ % sub_dict
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
273
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
274
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
275 def get_available_tables_for_dbkey( trans, dbkey, data_table_names ):
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
276 data_tables = load_data_tables_from_url( data_table_class=trans.app.tool_data_tables.__class__ )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
277 rval = {}
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
278 for name, data_table in data_tables.get( 'data_tables' ).get_tables().items():
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
279 if ( not data_table_names or name in data_table_names ):
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
280 # TODO: check that columns are similiar
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
281 if not dbkey:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
282 entry_getter = data_table.get_named_fields_list()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
283 else:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
284 entry_getter = data_table.get_entries( 'dbkey', dbkey, None, default=[] )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
285 for entry in entry_getter:
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
286 name = "%s: %s" % ( data_table.name, dumps( entry ) )
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
287 rval[name] = entry
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
288 return rval
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
289
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
290
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
291 def split_path_all( path ):
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
292 rval = []
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
293 path = path.rstrip( '/' )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
294 while True:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
295 head, tail = os.path.split( path )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
296 if tail:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
297 rval.append( tail )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
298 path = head
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
299 elif head:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
300 rval.append( head )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
301 break
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
302 else:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
303 break
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
304 rval.reverse()
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
305 return rval
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
306
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
307
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
308 def get_data_for_path( path, data_root_dir ):
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
309 # We list dir with a /, but copy data without
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
310 # listing with / gives a . entry when its a dir
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
311 # cloning without the / will copy that whole directory into the target,
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
312 # instead of just that target's contents
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
313 if path.startswith( GALAXY_DATA_CANONICAL_PATH ):
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
314 path = path[ len( GALAXY_DATA_CANONICAL_PATH ):]
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
315 make_path = path
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
316 rsync_source = rsync_urljoin( rsync_urljoin( RSYNC_SERVER, INDEX_DIR ), path )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
317 if rsync_source.endswith( '/' ):
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
318 rsync_source = rsync_source[:-1]
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
319 try:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
320 dir_list = rsync_list_dir( rsync_source + "/" )
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
321 except Exception:
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
322 dir_list = None
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
323 while not dir_list or '.' not in dir_list:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
324 head, tail = os.path.split( make_path )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
325 if not head:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
326 head = tail
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
327 make_path = head
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
328 rsync_source = rsync_urljoin( rsync_urljoin( RSYNC_SERVER, INDEX_DIR ), head ) # if we error here, likely due to a connection issue
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
329 if rsync_source.endswith( '/' ):
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
330 rsync_source = rsync_source[:-1]
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
331 dir_list = rsync_list_dir( rsync_source + "/" )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
332 split_path = split_path_all( make_path )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
333 target_path = data_root_dir
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
334 for p in split_path[:-1]:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
335 target_path = os.path.join( target_path, p )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
336 if not os.path.exists( target_path ):
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
337 os.mkdir( target_path )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
338 rsync_sync_to_dir( rsync_source, target_path )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
339 return path
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
340
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
341
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
342 def get_data_and_munge_path( data_table_name, data_table_entry, data_root_dir ):
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
343 path_cols = []
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
344 for key, value in data_table_entry.items():
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
345 if key in PATH_COLUMN_NAMES:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
346 path_cols.append( ( key, value ) )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
347 if path_cols:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
348 for col_name, value in path_cols:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
349 if value.startswith( GALAXY_DATA_CANONICAL_PATH ):
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
350 data_table_entry[col_name] = get_data_for_path( value, data_root_dir )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
351 else:
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
352 print('unable to determine location of rsync data for', data_table_name, data_table_entry)
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
353 return data_table_entry
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
354
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
355
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
356 def fulfill_data_table_entries( data_table_entries, data_manager_dict, data_root_dir ):
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
357 for data_table_name, entries in data_table_entries.items():
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
358 for entry in entries:
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
359 entry = get_data_and_munge_path( data_table_name, entry, data_root_dir )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
360 _add_data_table_entry( data_manager_dict, data_table_name, entry )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
361 return data_manager_dict
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
362
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
363
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
364 def _add_data_table_entry( data_manager_dict, data_table_name, data_table_entry ):
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
365 data_manager_dict['data_tables'] = data_manager_dict.get( 'data_tables', {} )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
366 data_manager_dict['data_tables'][data_table_name] = data_manager_dict['data_tables'].get( data_table_name, [] )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
367 data_manager_dict['data_tables'][data_table_name].append( data_table_entry )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
368 return data_manager_dict
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
369
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
370
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
371 def cleanup_before_exit( tmp_dir ):
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
372 if tmp_dir and os.path.exists( tmp_dir ):
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
373 shutil.rmtree( tmp_dir )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
374
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
375
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
376 def get_data_table_entries( params ):
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
377 rval = {}
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
378 data_table_entries = params.get( 'data_table_entries', None )
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
379 if data_table_entries:
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
380 for entry_text in data_table_entries.split( ',' ):
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
381 entry_text = entry_text.strip().decode( 'base64' )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
382 entry_dict = loads( entry_text )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
383 data_table_name = entry_dict['name']
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
384 data_table_entry = entry_dict['entry']
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
385 rval[ data_table_name ] = rval.get( data_table_name, [] )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
386 rval[ data_table_name ].append( data_table_entry )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
387 return rval
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
388
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
389
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
390 def main():
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
391 parser = optparse.OptionParser()
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
392 (options, args) = parser.parse_args()
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
393
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
394 filename = args[0]
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
395
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
396 params = loads( open( filename ).read() )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
397 target_directory = params[ 'output_data' ][0]['extra_files_path']
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
398 os.mkdir( target_directory )
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
399 data_manager_dict = {}
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
400
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
401 data_table_entries = get_data_table_entries( params['param_dict'] )
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
402
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
403 # Populate the data Tables
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
404 data_manager_dict = fulfill_data_table_entries( data_table_entries, data_manager_dict, target_directory )
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
405
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
406 # save info to json file
0
70afa70bba41 planemo upload for repository https://github.com/galaxyproject/tools-devteam/tree/master/data_managers/data_manager_rsync_g2 commit 86cf90107482cab1cb47fc0d42d6705f8077daa7
devteam
parents:
diff changeset
407 open( filename, 'wb' ).write( dumps( data_manager_dict ) )
1
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
408
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
409
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
410 if __name__ == "__main__":
861c071a6cd5 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
iuc
parents: 0
diff changeset
411 main()